Esempio n. 1
0
func load2Feed(cookie string) (feed *atom.Feed, err error) {
	data, err := getUrlContent(cookie, homePage)
	content := string(data)
	if err != nil {
		return
	}
	feed = &atom.Feed{Id: homePage, Title: "知乎", Entries: make([]atom.Entry, 0)}
	for {
		var url, title string
		var ok bool
		if url, content, ok = extract(content, `question_link" target="_blank" href="`, `"`); !ok {
			break
		}
		if strings.HasPrefix(url, "http://zhuanlan") {
			if url, content, ok = extract(content, `post-link" target="_blank" href="`, `"`); !ok {
				break
			}
		} else {
			url = "http://www.zhihu.com" + url
		}
		fmt.Println("got url:", url)
		if title, content, ok = extract(content, `>`, `</a>`); !ok {
			break
		}
		fmt.Println("got title:", title)
		hasEntry, err := db.HasEntry(feed.Id, url)
		if err != nil { //database query error
			return nil, err
		}
		if hasEntry { //found entry already in database
			continue
		}

		article := &Article{Url: url}
		err = article.parseArticle(cookie)
		if err == noAnswerErr {
			continue
		}
		if err != nil {
			return nil, err
		}
		entry := atom.Entry{Id: url, Title: title, Summary: article.Content}
		feed.Entries = append(feed.Entries, entry)
		err = db.SaveEntry(feed.Title, feed.Id, &entry)
		if err != nil {
			return nil, err
		}
	}
	return feed, nil
}
Esempio n. 2
0
func Update() error {
	var feed atom.Feed
	err := feed.Load(url)
	if err != nil {
		return err
	}

	if len(feed.Entries) < 1 {
		return errors.New("no entry found.")
	}

	var content string
	count := 0
	for _, entry := range feed.Entries {
		id := getItemQuestionId(entry.Id)
		var qs Questions
		err = question(&qs, id)
		if err != nil || len(qs.Items) < 1 {
			return errors.New("failure to request stackoverflow questions api")
		}
		if qs.Items[0].Accepted_answer_id > 0 {
			hasEntry, err := db.HasEntry(feed.Id, entry.Id)
			if err != nil { //database query error
				return err
			}
			if hasEntry { //found entry already in database
				continue
			}
			//found no entry
			var ans Answers
			err = answer(&ans, qs.Items[0].Accepted_answer_id)
			if err != nil || len(ans.Items) < 1 {
				return errors.New("failure to request stackoverflow answers api")
			}
			content = content + fmt.Sprintf(questionTemplate, qs.Items[0].Title, qs.Items[0].Body, ans.Items[0].Body)
			count = count + 1
			err = db.SaveEntry(feed.Title, feed.Id, &entry)
			if err != nil {
				return err
			}
		}
	}
	if count < 1 {
		return errors.New("no new feed found")
	}
	return write(content)
}
func load2Feed() (*atom.Feed, error) {
	resp, err := http.Get(kai_juan_ba_fen_zhong_url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	content := string(body)
	feed := &atom.Feed{Id: kai_juan_ba_fen_zhong_url, Title: "开卷八分钟", Entries: make([]atom.Entry, 0)}
	for {
		var url, title string
		var ok bool
		if url, content, ok = extract(content, `sys_url" href="`, `"`); !ok {
			break
		}
		if title, content, ok = extract(content, `>`, `</a>`); !ok {
			break
		}
		hasEntry, err := db.HasEntry(feed.Id, url)
		if err != nil { //database query error
			return nil, err
		}
		if hasEntry { //found entry already in database
			continue
		}

		article := &Article{Url: url}
		err = article.parseArticle()
		if err != nil {
			return nil, err
		}
		entry := atom.Entry{Id: url, Title: title, Summary: article.Content}
		feed.Entries = append(feed.Entries, entry)
		err = db.SaveEntry(feed.Title, feed.Id, &entry)
		if err != nil {
			return nil, err
		}
	}
	return feed, nil
}