// GroupScanForward looks for new messages in a particular Group.
// Returns the number of articles scanned and if an error was encountered
func (n *NNTPClient) GroupScanForward(dbh *db.Handle, group string, limit int) (int, error) {
ctxLogger := logrus.WithFields(
logrus.Fields{
"group": group,
},
)
nntpGroup, err := n.c.Group(group)
if err != nil {
return 0, err
}
g, err := dbh.FindGroupByName(group)
if err != nil {
return 0, err
}
if g.First == 0 {
ctxLogger.Infof("DB Group First not set, setting to lowest message in group: %d", nntpGroup.Low)
g.First = nntpGroup.Low
}
if g.Last == 0 {
if limit > 0 {
ctxLogger.Infof("DB Group Last seen not set, setting to most recent message minus max to fetch: %d", nntpGroup.High-int64(limit))
g.Last = nntpGroup.High - int64(limit)
} else {
ctxLogger.Infof("DB Group Last seen not set and no limit given, using default of %d", defaultMaxOverview)
g.Last = nntpGroup.High - defaultMaxOverview
if g.Last < nntpGroup.Low {
g.Last = nntpGroup.Low
}
}
}
if g.First < nntpGroup.Low {
ctxLogger.Infof("Group %s first article was older than first on server (%d < %d), resetting to %d", g.Name, g.First, nntpGroup.Low, nntpGroup.Low)
g.First = nntpGroup.Low
}
if g.Last > nntpGroup.High {
ctxLogger.Errorf("Group %s last article is newer than on server (%d > %d), resetting to %d.", g.Name, g.Last, nntpGroup.High, nntpGroup.High)
g.Last = nntpGroup.High
}
err = dbh.DB.Save(g).Error
if err != nil {
return 0, err
}
newMessages := nntpGroup.High - g.Last
var maxToGet int64
if limit > 0 {
maxToGet = g.Last + int64(limit)
} else {
maxToGet = nntpGroup.High
}
if maxToGet > nntpGroup.High {
maxToGet = nntpGroup.High
}
if newMessages < 1 {
ctxLogger.Info("No new articles")
return 0, nil
}
ctxLogger.Infof("%d new articles limited to getting just %d (%d - %d) in %d article chunks", newMessages, maxToGet-g.Last, g.Last, maxToGet, n.MaxScan)
begin := g.Last + 1
totalArticles := 0
missedMessages := 0
remaining := newMessages
for begin < maxToGet {
toGet := begin + int64(n.MaxScan) - 1
if toGet > maxToGet {
toGet = maxToGet
}
if toGet < begin {
toGet = begin
}
remaining = remaining - toGet
ctxLogger.Debugf("Getting %d-%d (%d remaining)", begin, toGet, remaining)
overviews, err := n.c.Overview(begin, toGet)
if err != nil {
return len(overviews), err
}
var mm types.MessageNumberSet
if n.SaveMissed {
mm = findMissingMessages(begin, toGet, overviews)
missedMessages = missedMessages + mm.Cardinality()
ctxLogger.Debugf("Got %d messages and %d missed messages", len(overviews), mm.Cardinality())
} else {
ctxLogger.Debugf("Got %d messages", len(overviews))
}
totalArticles = totalArticles + len(overviews)
ctxLogger.Debugf("Saving parts and messages to db.")
err = saveOverviewBatch(dbh, g.Name, overviews, mm)
if err != nil {
return len(overviews), err
}
begin = toGet + 1
g.Last = toGet
err = dbh.DB.Save(g).Error
if err != nil {
return 0, err
}
}
if n.SaveMissed {
ctxLogger.Infof("Got %d messages and %d missed messages", totalArticles, missedMessages)
} else {
ctxLogger.Debugf("Got %d messages", totalArticles)
}
err = dbh.DB.Save(g).Error
if err != nil {
return 0, err
}
return totalArticles, nil
}
