// Translated from nzedb Binaries.php and pynab handling
// Save all messages that match a basic regex as segments and
func saveOverviewBatch(dbh *db.Handle, group string, overviews []nntp.MessageOverview, missed types.MessageNumberSet) error {
parts := map[string]*types.Part{}
for _, o := range overviews {
//TODO: make this a function call
if blacklistRegexp.MatchString(o.Subject) {
continue
}
m := segmentRegexp.FindStringSubmatch(o.Subject)
if m != nil {
subj := m[1]
if !yencRegexp.MatchString(subj) {
subj += " yEnc"
}
segNum, _ := strconv.Atoi(m[2])
segTotal, _ := strconv.Atoi(m[3])
hash := hashOverview(subj, o.From, group, segTotal)
seg := types.Segment{
MessageID: o.MessageID,
Segment: segNum,
Size: int64(o.Bytes),
}
if part, ok := parts[hash]; ok {
part.Segments = append(part.Segments, seg)
} else {
parts[hash] = &types.Part{
Hash: hash,
Subject: subj,
Posted: o.Date,
From: o.From,
GroupName: group,
TotalSegments: segTotal,
Xref: o.Xref(),
Segments: []types.Segment{seg},
}
}
}
}
mm := make([]types.MissedMessage, missed.Cardinality())
i := 0
for id := range missed.Iter() {
mm[i] = types.MissedMessage{
MessageNumber: int64(id),
GroupName: group,
Attempts: 1,
}
i++
}
logrus.Debugf("Found %d new parts, %d missed messages", len(parts), len(mm))
return dbh.SavePartsAndMissedMessages(parts, mm)
}
// GroupScanForward looks for new messages in a particular Group.
// Returns the number of articles scanned and if an error was encountered
func (n *NNTPClient) GroupScanForward(dbh *db.Handle, group string, limit int) (int, error) {
ctxLogger := logrus.WithFields(
logrus.Fields{
"group": group,
},
)
nntpGroup, err := n.c.Group(group)
if err != nil {
return 0, err
}
g, err := dbh.FindGroupByName(group)
if err != nil {
return 0, err
}
if g.First == 0 {
ctxLogger.Infof("DB Group First not set, setting to lowest message in group: %d", nntpGroup.Low)
g.First = nntpGroup.Low
}
if g.Last == 0 {
if limit > 0 {
ctxLogger.Infof("DB Group Last seen not set, setting to most recent message minus max to fetch: %d", nntpGroup.High-int64(limit))
g.Last = nntpGroup.High - int64(limit)
} else {
ctxLogger.Infof("DB Group Last seen not set and no limit given, using default of %d", defaultMaxOverview)
g.Last = nntpGroup.High - defaultMaxOverview
if g.Last < nntpGroup.Low {
g.Last = nntpGroup.Low
}
}
}
if g.First < nntpGroup.Low {
ctxLogger.Infof("Group %s first article was older than first on server (%d < %d), resetting to %d", g.Name, g.First, nntpGroup.Low, nntpGroup.Low)
g.First = nntpGroup.Low
}
if g.Last > nntpGroup.High {
ctxLogger.Errorf("Group %s last article is newer than on server (%d > %d), resetting to %d.", g.Name, g.Last, nntpGroup.High, nntpGroup.High)
g.Last = nntpGroup.High
}
err = dbh.DB.Save(g).Error
if err != nil {
return 0, err
}
newMessages := nntpGroup.High - g.Last
var maxToGet int64
if limit > 0 {
maxToGet = g.Last + int64(limit)
} else {
maxToGet = nntpGroup.High
}
if maxToGet > nntpGroup.High {
maxToGet = nntpGroup.High
}
if newMessages < 1 {
ctxLogger.Info("No new articles")
return 0, nil
}
ctxLogger.Infof("%d new articles limited to getting just %d (%d - %d) in %d article chunks", newMessages, maxToGet-g.Last, g.Last, maxToGet, n.MaxScan)
begin := g.Last + 1
totalArticles := 0
missedMessages := 0
remaining := newMessages
for begin < maxToGet {
toGet := begin + int64(n.MaxScan) - 1
if toGet > maxToGet {
toGet = maxToGet
}
if toGet < begin {
toGet = begin
}
remaining = remaining - toGet
ctxLogger.Debugf("Getting %d-%d (%d remaining)", begin, toGet, remaining)
overviews, err := n.c.Overview(begin, toGet)
if err != nil {
return len(overviews), err
}
var mm types.MessageNumberSet
if n.SaveMissed {
mm = findMissingMessages(begin, toGet, overviews)
missedMessages = missedMessages + mm.Cardinality()
ctxLogger.Debugf("Got %d messages and %d missed messages", len(overviews), mm.Cardinality())
} else {
ctxLogger.Debugf("Got %d messages", len(overviews))
}
totalArticles = totalArticles + len(overviews)
ctxLogger.Debugf("Saving parts and messages to db.")
err = saveOverviewBatch(dbh, g.Name, overviews, mm)
if err != nil {
return len(overviews), err
}
begin = toGet + 1
g.Last = toGet
err = dbh.DB.Save(g).Error
if err != nil {
return 0, err
}
}
if n.SaveMissed {
ctxLogger.Infof("Got %d messages and %d missed messages", totalArticles, missedMessages)
} else {
ctxLogger.Debugf("Got %d messages", totalArticles)
}
err = dbh.DB.Save(g).Error
if err != nil {
return 0, err
}
return totalArticles, nil
}