// CheckUrl returns information about a feed and nil if the URL is a
// valid feed, otherwise it returns an error.
func checkUrl(c appengine.Context, url string) (FeedInfo, error) {
resp, err := urlfetch.Client(c).Get(url)
if err != nil {
return FeedInfo{}, err
}
defer resp.Body.Close()
f, err := webfeed.Read(resp.Body)
if err != nil {
if _, ok := err.(webfeed.ErrBadTime); ok {
c.Infof("Ignoring bad time format for feed %s: %s", url, err.Error())
err = nil
} else {
return FeedInfo{}, err
}
}
return FeedInfo{Url: url, Title: f.Title, Link: f.Link}, err
}
// FetchUrl reads a feed from the given URL.
func fetchUrl(c appengine.Context, url string) (FeedInfo, Articles, error) {
var finfo FeedInfo
resp, err := urlfetch.Client(c).Get(url)
if err != nil {
return finfo, nil, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return finfo, nil, errors.New("HTTP error: " + http.StatusText(resp.StatusCode))
}
defer resp.Body.Close()
feed, err := webfeed.Read(resp.Body)
if err != nil {
if _, ok := err.(webfeed.ErrBadTime); ok {
c.Infof("Ignoring bad time format for feed %s: %s", url, err.Error())
err = nil
} else {
err = errors.New("failed to fetch " + url + ": " + err.Error())
return finfo, nil, err
}
}
finfo.Url = url
finfo.Title = feed.Title
if finfo.Title == "" {
finfo.Title = url
}
finfo.Link = feed.Link
finfo.LastFetch = time.Now()
as := make(Articles, len(feed.Entries))
for i, ent := range feed.Entries {
content := ent.Content
if len(ent.Content) == 0 {
content = ent.Summary
}
title := ent.Title
if title == "" && len(ent.Summary) > 0 {
n := len(ent.Summary)
if n > 20 {
n = 20
}
title = string(ent.Summary[:n]) + "…"
}
if title == "" {
title = ent.Link
}
id := ent.ID
if id == "" {
id = ent.Title + ";" + ent.When.Format("06-01-02;15:04:05")
}
as[i] = Article{
Title: title,
Link: ent.Link,
OriginTitle: feed.Title,
DescriptionData: content,
When: ent.When,
id: id,
}
}
return finfo, as, nil
}