示例#1
0
func (fm *FeedManager) requestFeedContent(f content.Feed) {
	if f == nil {
		fm.logger.Infoln("No feed provided")
		return
	}

	data := f.Data()

	fm.logger.Infoln("Requesting feed content for " + f.String())

	resp, err := fm.client.Get(data.Link)

	if err != nil {
		data.UpdateError = err.Error()
	} else if resp.StatusCode != http.StatusOK {
		defer func() {
			// Drain the body so that the connection can be reused
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}()
		data.UpdateError = httpStatusPrefix + strconv.Itoa(resp.StatusCode)
	} else {
		defer resp.Body.Close()
		data.UpdateError = ""

		buf := util.BufferPool.GetBuffer()
		defer util.BufferPool.Put(buf)

		if _, err := buf.ReadFrom(resp.Body); err == nil {
			hash := md5.Sum(buf.Bytes())
			if b, ok := fm.lastUpdateHash[data.Id]; ok && bytes.Equal(b[:], hash[:]) {
				fm.logger.Infof("Content of feed %s is the same as the previous update\n", f)
				return
			}
			fm.lastUpdateHash[data.Id] = hash

			if pf, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
				f.Refresh(fm.processParserFeed(pf))
			} else {
				data.UpdateError = err.Error()
			}
		} else {
			data.UpdateError = err.Error()
		}

	}

	if data.UpdateError != "" {
		fm.logger.Printf("Error updating feed '%s': %s\n", f, data.UpdateError)
	}

	f.Data(data)

	select {
	case <-fm.done:
		return
	default:
		fm.updateFeed(f)
	}
}
示例#2
0
func (fm *FeedManager) requestFeedContent(f Feed) Feed {
	Debug.Println("Requesting feed content for " + f.Link)

	resp, err := fm.client.Get(f.Link)

	if err != nil {
		f.UpdateError = err.Error()
	} else if resp.StatusCode != http.StatusOK {
		f.UpdateError = httpStatusPrefix + strconv.Itoa(resp.StatusCode)
	} else {
		f.UpdateError = ""

		buf := util.BufferPool.GetBuffer()
		defer util.BufferPool.Put(buf)

		if _, err := buf.ReadFrom(resp.Body); err == nil {
			if pf, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
				f = f.UpdateFromParsed(pf)
			} else {
				f.UpdateError = err.Error()
			}
		} else {
			f.UpdateError = err.Error()
		}

	}

	if f.UpdateError != "" {
		fm.logger.Printf("Error updating feed: %s\n", f.UpdateError)
	}

	select {
	case <-fm.done:
		return f
	default:
		f, newArticles, err := fm.db.UpdateFeed(f)
		if err != nil {
			fm.logger.Printf("Error updating feed database record: %v\n", err)
		}

		if newArticles {
			fm.updateFeed <- f
		}

		return f
	}
}
示例#3
0
文件: hubbub.go 项目: RavenB/readeef
func (con HubbubController) Handler(c context.Context) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		params := r.URL.Query()
		pathParams := webfw.GetParams(c, r)
		logger := webfw.GetLogger(c)
		feedId, err := strconv.ParseInt(pathParams["feed-id"], 10, 64)

		if err != nil {
			webfw.GetLogger(c).Print(err)
			return
		}

		repo := con.hubbub.repo
		f := repo.FeedById(data.FeedId(feedId))
		s := f.Subscription()

		err = s.Err()

		if err != nil {
			webfw.GetLogger(c).Print(err)
			return
		}

		logger.Infoln("Receiving hubbub event " + params.Get("hub.mode") + " for " + f.String())

		data := s.Data()
		switch params.Get("hub.mode") {
		case "subscribe":
			if lease, err := strconv.Atoi(params.Get("hub.lease_seconds")); err == nil {
				data.LeaseDuration = int64(lease) * int64(time.Second)
			}
			data.VerificationTime = time.Now()

			w.Write([]byte(params.Get("hub.challenge")))
		case "unsubscribe":
			w.Write([]byte(params.Get("hub.challenge")))
		case "denied":
			w.Write([]byte{})
			webfw.GetLogger(c).Printf("Unable to subscribe to '%s': %s\n", params.Get("hub.topic"), params.Get("hub.reason"))
		default:
			w.Write([]byte{})

			buf := util.BufferPool.GetBuffer()
			defer util.BufferPool.Put(buf)

			if _, err := buf.ReadFrom(r.Body); err != nil {
				webfw.GetLogger(c).Print(err)
				return
			}

			newArticles := false

			if pf, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
				f.Refresh(pf)
				f.Update()

				if f.HasErr() {
					webfw.GetLogger(c).Print(f.Err())
					return
				}

				newArticles = len(f.NewArticles()) > 0
			} else {
				webfw.GetLogger(c).Print(err)
				return
			}

			if newArticles {
				con.hubbub.NotifyReceivers(f)
			}

			return
		}

		switch params.Get("hub.mode") {
		case "subscribe":
			data.SubscriptionFailure = false
		case "unsubscribe", "denied":
			data.SubscriptionFailure = true
		}

		s.Data(data)
		s.Update()
		if s.HasErr() {
			webfw.GetLogger(c).Print(s.Err())
			return
		}

		if data.SubscriptionFailure {
			con.hubbub.removeFeed <- f
		} else {
			con.hubbub.addFeed <- f
		}
	})
}
示例#4
0
文件: hubbub.go 项目: urandom/readeef
func (con HubbubController) Handler(c context.Context) http.Handler {
	logger := webfw.GetLogger(c)
	repo := readeef.GetRepo(c)

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		params := r.URL.Query()
		pathParams := webfw.GetParams(c, r)
		feedId, err := strconv.ParseInt(pathParams["feed-id"], 10, 64)

		if err != nil {
			logger.Print(err)
			return
		}

		f := repo.FeedById(data.FeedId(feedId))
		s := f.Subscription()

		err = s.Err()

		if err != nil {
			logger.Print(err)
			return
		}

		logger.Infoln("Receiving hubbub event " + params.Get("hub.mode") + " for " + f.String())

		data := s.Data()
		switch params.Get("hub.mode") {
		case "subscribe":
			if lease, err := strconv.Atoi(params.Get("hub.lease_seconds")); err == nil {
				data.LeaseDuration = int64(lease) * int64(time.Second)
			}
			data.VerificationTime = time.Now()

			w.Write([]byte(params.Get("hub.challenge")))
		case "unsubscribe":
			// Nothing to do here, the subscription will be removed along with the feed by the manager
			w.Write([]byte(params.Get("hub.challenge")))
		case "denied":
			w.Write([]byte{})
			logger.Printf("Unable to subscribe to '%s': %s\n", params.Get("hub.topic"), params.Get("hub.reason"))
		default:
			w.Write([]byte{})

			buf := util.BufferPool.GetBuffer()
			defer util.BufferPool.Put(buf)

			if _, err := buf.ReadFrom(r.Body); err != nil {
				logger.Print(err)
				return
			}

			newArticles := false

			if pf, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
				f.Refresh(pf)
				f.Update()

				if f.HasErr() {
					logger.Print(f.Err())
					return
				}

				newArticles = len(f.NewArticles()) > 0
			} else {
				logger.Print(err)
				return
			}

			if newArticles {
				for _, m := range con.hubbub.FeedMonitors() {
					if err := m.FeedUpdated(f); err != nil {
						logger.Printf("Error invoking monitor '%s' on updated feed '%s': %v\n",
							reflect.TypeOf(m), f, err)
					}
				}
			}

			return
		}

		switch params.Get("hub.mode") {
		case "subscribe":
			data.SubscriptionFailure = false
		case "unsubscribe", "denied":
			data.SubscriptionFailure = true
		}

		s.Data(data)
		s.Update()
		if s.HasErr() {
			logger.Print(fmt.Errorf("Error updating subscription %s: %v\n", s, s.Err()))
			return
		}

		if data.SubscriptionFailure {
			con.removeFeed <- f
		} else {
			con.addFeed <- f
		}
	})
}
示例#5
0
func discoverParserFeeds(link string) ([]Feed, error) {
	resp, err := http.Get(link)
	if err != nil {
		return []Feed{}, err
	}

	buf := util.BufferPool.GetBuffer()
	defer util.BufferPool.Put(buf)

	buf.ReadFrom(resp.Body)

	if parserFeed, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
		feed := Feed{Link: link}
		feed = feed.UpdateFromParsed(parserFeed)

		return []Feed{feed}, nil
	} else {
		html := commentPattern.ReplaceAllString(buf.String(), "")
		links := linkPattern.FindAllStringSubmatch(html, -1)

		feeds := []Feed{}
		for _, l := range links {
			attrs := l[1]
			if strings.Contains(attrs, `"application/rss+xml"`) || strings.Contains(attrs, `'application/rss+xml'`) {
				index := strings.Index(attrs, "href=")
				attr := attrs[index+6:]
				index = strings.IndexByte(attr, attrs[index+5])
				href := attr[:index]

				if u, err := url.Parse(href); err != nil {
					return []Feed{}, err
				} else {
					if !u.IsAbs() {
						l, _ := url.Parse(link)

						u.Scheme = l.Scheme

						if u.Host == "" {
							u.Host = l.Host
						}

						href = u.String()
					}

					Debug.Printf("Checking if '%s' is a valid feed link\n", href)

					fs, err := discoverParserFeeds(href)
					if err != nil {
						return []Feed{}, err
					}

					feeds = append(feeds, fs[0])
				}
			}
		}

		if len(feeds) != 0 {
			return feeds, nil
		}
	}

	return []Feed{}, ErrNoFeed
}
示例#6
0
func (con HubbubController) Handler(c context.Context) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()

		params := r.URL.Query()
		pathParams := webfw.GetParams(c, r)
		feedId, err := strconv.ParseInt(pathParams["feed-id"], 10, 64)

		if err != nil {
			webfw.GetLogger(c).Print(err)
			return
		}

		s, err := con.hubbub.db.GetHubbubSubscription(feedId)

		if err != nil {
			webfw.GetLogger(c).Print(err)
			return
		}

		f, err := con.hubbub.db.GetFeed(s.FeedId)
		if err != nil {
			webfw.GetLogger(c).Print(err)
			return
		}

		Debug.Println("Receiving hubbub event " + params.Get("hub.mode") + " for " + f.Link)

		switch params.Get("hub.mode") {
		case "subscribe":
			if lease, err := strconv.Atoi(params.Get("hub.lease_seconds")); err == nil {
				s.LeaseDuration = int64(lease) * int64(time.Second)
			}
			s.VerificationTime = time.Now()

			w.Write([]byte(params.Get("hub.challenge")))
		case "unsubscribe":
			w.Write([]byte(params.Get("hub.challenge")))
		case "denied":
			w.Write([]byte{})
			webfw.GetLogger(c).Printf("Unable to subscribe to '%s': %s\n", params.Get("hub.topic"), params.Get("hub.reason"))
		default:
			w.Write([]byte{})

			buf := util.BufferPool.GetBuffer()
			defer util.BufferPool.Put(buf)

			if _, err := buf.ReadFrom(r.Body); err != nil {
				webfw.GetLogger(c).Print(err)
				return
			}

			newArticles := false

			if pf, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
				f = f.UpdateFromParsed(pf)

				_, newArticles, err = con.hubbub.db.UpdateFeed(f)
				if err != nil {
					webfw.GetLogger(c).Print(err)
					return
				}
			} else {
				webfw.GetLogger(c).Print(err)
				return
			}

			if newArticles {
				con.hubbub.updateFeed <- f
			}

			return
		}

		switch params.Get("hub.mode") {
		case "subscribe":
			s.SubscriptionFailure = false
		case "unsubscribe", "denied":
			s.SubscriptionFailure = true
		}

		if err := con.hubbub.db.UpdateHubbubSubscription(s); err != nil {
			webfw.GetLogger(c).Print(err)
			return
		}

		if s.SubscriptionFailure {
			con.hubbub.removeFeed <- f
		} else {
			con.hubbub.addFeed <- f
		}
	}
}
示例#7
0
func (fm FeedManager) discoverParserFeeds(link string) ([]content.Feed, error) {
	fm.logger.Debugf("Fetching feed link body %s\n", link)
	resp, err := http.Get(link)
	if err != nil {
		return []content.Feed{}, err
	}
	defer resp.Body.Close()

	buf := util.BufferPool.GetBuffer()
	defer util.BufferPool.Put(buf)

	buf.ReadFrom(resp.Body)

	if parserFeed, err := parser.ParseFeed(buf.Bytes(), parser.ParseRss2, parser.ParseAtom, parser.ParseRss1); err == nil {
		fm.logger.Debugf("Discovering link %s contains feed data\n", link)

		feed := fm.repo.Feed()

		feed.Data(data.Feed{Link: link})
		feed.Refresh(fm.processParserFeed(parserFeed))

		return []content.Feed{feed}, nil
	} else {
		fm.logger.Debugf("Searching for html links within the discovering link %s\n", link)

		html := commentPattern.ReplaceAllString(buf.String(), "")
		links := linkPattern.FindAllStringSubmatch(html, -1)

		feeds := []content.Feed{}
		for _, l := range links {
			attrs := l[1]
			if strings.Contains(attrs, `"application/rss+xml"`) || strings.Contains(attrs, `'application/rss+xml'`) {
				index := strings.Index(attrs, "href=")
				attr := attrs[index+6:]
				index = strings.IndexByte(attr, attrs[index+5])
				href := attr[:index]

				if u, err := url.Parse(href); err != nil {
					return []content.Feed{}, ErrNoFeed
				} else {
					if !u.IsAbs() {
						l, _ := url.Parse(link)

						u.Scheme = l.Scheme

						if u.Host == "" {
							u.Host = l.Host
						}

						href = u.String()
					}

					fs, err := fm.discoverParserFeeds(href)
					if err != nil {
						return []content.Feed{}, err
					}

					feeds = append(feeds, fs[0])
				}
			}
		}

		if len(feeds) != 0 {
			return feeds, nil
		}
	}

	return []content.Feed{}, ErrNoFeed
}