// GroupScanForward looks for new messages in a particular Group. // Returns the number of articles scanned and if an error was encountered func (n *NNTPClient) GroupScanForward(dbh *db.Handle, group string, limit int) (int, error) { ctxLogger := logrus.WithFields( logrus.Fields{ "group": group, }, ) nntpGroup, err := n.c.Group(group) if err != nil { return 0, err } g, err := dbh.FindGroupByName(group) if err != nil { return 0, err } if g.First == 0 { ctxLogger.Infof("DB Group First not set, setting to lowest message in group: %d", nntpGroup.Low) g.First = nntpGroup.Low } if g.Last == 0 { if limit > 0 { ctxLogger.Infof("DB Group Last seen not set, setting to most recent message minus max to fetch: %d", nntpGroup.High-int64(limit)) g.Last = nntpGroup.High - int64(limit) } else { ctxLogger.Infof("DB Group Last seen not set and no limit given, using default of %d", defaultMaxOverview) g.Last = nntpGroup.High - defaultMaxOverview if g.Last < nntpGroup.Low { g.Last = nntpGroup.Low } } } if g.First < nntpGroup.Low { ctxLogger.Infof("Group %s first article was older than first on server (%d < %d), resetting to %d", g.Name, g.First, nntpGroup.Low, nntpGroup.Low) g.First = nntpGroup.Low } if g.Last > nntpGroup.High { ctxLogger.Errorf("Group %s last article is newer than on server (%d > %d), resetting to %d.", g.Name, g.Last, nntpGroup.High, nntpGroup.High) g.Last = nntpGroup.High } err = dbh.DB.Save(g).Error if err != nil { return 0, err } newMessages := nntpGroup.High - g.Last var maxToGet int64 if limit > 0 { maxToGet = g.Last + int64(limit) } else { maxToGet = nntpGroup.High } if maxToGet > nntpGroup.High { maxToGet = nntpGroup.High } if newMessages < 1 { ctxLogger.Info("No new articles") return 0, nil } ctxLogger.Infof("%d new articles limited to getting just %d (%d - %d) in %d article chunks", newMessages, maxToGet-g.Last, g.Last, maxToGet, n.MaxScan) begin := g.Last + 1 totalArticles := 0 missedMessages := 0 remaining := newMessages for begin < maxToGet { toGet := begin + int64(n.MaxScan) - 1 if toGet > maxToGet { toGet = maxToGet } if toGet < begin { toGet = begin } remaining = remaining - toGet ctxLogger.Debugf("Getting %d-%d (%d remaining)", begin, toGet, remaining) overviews, err := n.c.Overview(begin, toGet) if err != nil { return len(overviews), err } var mm types.MessageNumberSet if n.SaveMissed { mm = findMissingMessages(begin, toGet, overviews) missedMessages = missedMessages + mm.Cardinality() ctxLogger.Debugf("Got %d messages and %d missed messages", len(overviews), mm.Cardinality()) } else { ctxLogger.Debugf("Got %d messages", len(overviews)) } totalArticles = totalArticles + len(overviews) ctxLogger.Debugf("Saving parts and messages to db.") err = saveOverviewBatch(dbh, g.Name, overviews, mm) if err != nil { return len(overviews), err } begin = toGet + 1 g.Last = toGet err = dbh.DB.Save(g).Error if err != nil { return 0, err } } if n.SaveMissed { ctxLogger.Infof("Got %d messages and %d missed messages", totalArticles, missedMessages) } else { ctxLogger.Debugf("Got %d messages", totalArticles) } err = dbh.DB.Save(g).Error if err != nil { return 0, err } return totalArticles, nil }