Example #1
0
// download all new posts from a remote server
func (s *Server) downloadPosts(cfg *config.FeedConfig) error {
	dialer := network.NewDialer(cfg.Proxy)
	c, err := dialer.Dial(cfg.Addr)
	if err != nil {
		return err
	}
	conn := newOutboundConn(c, s, cfg)
	err = conn.Negotiate(false)
	if err != nil {
		conn.Quit()
		return err
	}
	groups, err := conn.ListNewsgroups()
	if err != nil {
		conn.Quit()
		return err
	}
	for _, g := range groups {
		if cfg.Policy != nil && cfg.Policy.AllowGroup(g.String()) {
			log.WithFields(log.Fields{
				"group": g,
				"pkg":   "nntp-server",
			}).Debug("downloading group")
			err = conn.DownloadGroup(g)
			if err != nil {
				conn.Quit()
				return err
			}
		}
	}
	conn.Quit()
	return nil
}
Example #2
0
// persist 1 feed forever
func (s *Server) persist(cfg *config.FeedConfig) {
	delay := time.Second

	log.WithFields(log.Fields{
		"name": cfg.Name,
	}).Debug("Persist Feed")
	for {
		dialer := network.NewDialer(cfg.Proxy)
		c, err := dialer.Dial(cfg.Addr)
		if err == nil {
			// successful connect
			delay = time.Second
			conn := newOutboundConn(c, s, cfg)
			err = conn.Negotiate(true)
			if err == nil {
				// negotiation good
				log.WithFields(log.Fields{
					"name": cfg.Name,
				}).Debug("Negotitation good")
				// start streaming
				var chnl chan ArticleEntry
				chnl, err = conn.StartStreaming()
				if err == nil {
					// register new connection
					f := &nntpFeed{
						conn: conn,
						send: chnl,
						conf: cfg,
					}
					s.regis <- f
					// start streaming
					conn.StreamAndQuit()
					// deregister
					s.deregis <- f
					continue
				}
			} else {
				log.WithFields(log.Fields{
					"name": cfg.Name,
				}).Info("outbound nntp connection failed to negotiate ", err)
			}
			conn.Quit()
		} else {
			// failed dial, do exponential backoff up to 1 hour
			if delay <= time.Hour {
				delay *= 2
			}
			log.WithFields(log.Fields{
				"name": cfg.Name,
			}).Info("feed backoff for ", delay)
			time.Sleep(delay)
		}
	}
}