func fetch_entries(feds []meta.Feed, done chan<- int) {
fmt.Println("fetch entries", len(feds))
curl := d.DefaultCurl()
for _, feed := range feds {
rsfile, _, _, err := curl.Download(feed.Link)
fmt.Println(rsfile, err)
if err != nil {
fmt.Println(err)
continue
}
f, entries, err := feeds.NewFeed(rsfile, feed.Link)
if err != nil {
fmt.Println(feed.Link, len(entries), err)
feeds.NewFeedOperator().Disable(feed.Link, true)
} else {
fmt.Println(f.Title, len(entries))
}
if err == nil {
ins, _ := feeds.NewEntryOperator().Save2(entries)
fmt.Println(len(ins), feed.Link)
}
}
done <- 0
}
func touch_entry(echan <-chan *meta.Entry, done chan<- int) {
for e := <-echan; e != nil; e = <-echan {
tf, sc, _ := d.ExtractHtml(e.Link)
// cf := feeds.NewContentFile(tf, sc.Words, sc.Imgs)
feeds.NewEntryOperator().SetContent(e.Link, tf, sc.WordCount, sc.Images)
// feeds.EntryUpdateContent(tf, e.Link)
// feeds.FetchEntryImagesExternal(e)
}
done <- 0
}
func feeds_diag(w http.ResponseWriter, r *http.Request) {
log.Println(r.URL)
eo := feeds.NewEntryOperator()
entries, err := eo.TopN(0, 10)
if err != nil {
error_json(w, new_error(err), http.StatusBadGateway)
// http.Error(w, err, http.StatusBadGateway)
return
}
write_json(w, entries)
// index(w, r)
}
func main() {
c, items, _ := feeds.NewFeed(rssfile, base_url)
fmt.Println(c)
for _, item := range items {
fmt.Println(item.Title)
}
fo := feeds.NewFeedOperator()
fo.Upsert(c)
x, _ := fo.Find("http://fulltextrssfeed.com/www.infzm.com/rss/home/rss2.0.xml")
fmt.Println(x)
eo := feeds.NewEntryOperator()
eo.Save(items)
// feeds.InsertChannel(c)
// test_db(items)
}
func main() {
defer recover_panic()
flag.Parse()
if len(*uri) == 0 {
fmt.Println(errors.New("entry_fetcher --uri http://feeds.feedburner.com/chinadigitaltimes/ywgz"))
return
}
log.Println("fetching", *uri)
rsfile, _, _, _ := d.DownloadFile(*uri)
log.Println("fetch url to ", rsfile)
f, entries, err := feeds.NewFeed(rsfile, *uri)
log.Println(f, len(entries), err)
feeds.NewEntryOperator().Save(entries)
}
func main() {
/*
fo := feeds.NewFeedOperator()
fds, err := fo.TimeoutFeeds()
fmt.Println(len(fds), err)
for _, feed := range fds {
err := fo.Touch(feed.Link, feed.TTL)
if err != nil {
fmt.Println(err)
}
}
*/
eo := feeds.NewEntryOperator()
etris, err := eo.TopN(0, 10)
fmt.Println(len(etris), err)
for _, entry := range etris {
err = eo.SetContent(entry.Link, "", 1, []string{"a.dumy", "b.dmy"})
}
}