// download all new posts from a remote server
func (s *Server) downloadPosts(cfg *config.FeedConfig) error {
dialer := network.NewDialer(cfg.Proxy)
c, err := dialer.Dial(cfg.Addr)
if err != nil {
return err
}
conn := newOutboundConn(c, s, cfg)
err = conn.Negotiate(false)
if err != nil {
conn.Quit()
return err
}
groups, err := conn.ListNewsgroups()
if err != nil {
conn.Quit()
return err
}
for _, g := range groups {
if cfg.Policy != nil && cfg.Policy.AllowGroup(g.String()) {
log.WithFields(log.Fields{
"group": g,
"pkg": "nntp-server",
}).Debug("downloading group")
err = conn.DownloadGroup(g)
if err != nil {
conn.Quit()
return err
}
}
}
conn.Quit()
return nil
}
// persist 1 feed forever
func (s *Server) persist(cfg *config.FeedConfig) {
delay := time.Second
log.WithFields(log.Fields{
"name": cfg.Name,
}).Debug("Persist Feed")
for {
dialer := network.NewDialer(cfg.Proxy)
c, err := dialer.Dial(cfg.Addr)
if err == nil {
// successful connect
delay = time.Second
conn := newOutboundConn(c, s, cfg)
err = conn.Negotiate(true)
if err == nil {
// negotiation good
log.WithFields(log.Fields{
"name": cfg.Name,
}).Debug("Negotitation good")
// start streaming
var chnl chan ArticleEntry
chnl, err = conn.StartStreaming()
if err == nil {
// register new connection
f := &nntpFeed{
conn: conn,
send: chnl,
conf: cfg,
}
s.regis <- f
// start streaming
conn.StreamAndQuit()
// deregister
s.deregis <- f
continue
}
} else {
log.WithFields(log.Fields{
"name": cfg.Name,
}).Info("outbound nntp connection failed to negotiate ", err)
}
conn.Quit()
} else {
// failed dial, do exponential backoff up to 1 hour
if delay <= time.Hour {
delay *= 2
}
log.WithFields(log.Fields{
"name": cfg.Name,
}).Info("feed backoff for ", delay)
time.Sleep(delay)
}
}
}