func EntityToLink(body string) string {
urls := text.ExtractURLs(body)
for _, url := range urls {
new := fmt.Sprintf("<a href=\"%s\">%s</a>", url, url)
body = strings.Replace(body, url, new, -1)
}
tags := text.ExtractHashtags(body)
for _, tag := range tags {
new := fmt.Sprintf("<a href=\"/hashtag/%s\">%s</a>", tag, tag)
body = strings.Replace(body, tag, new, -1)
}
return body
}
func (fa *FeedAgent) fetchService(job *pb.FeedJob) (int, error) {
stream, err := fa.client.ArchiveFeed(context.Background())
defer stream.CloseAndRecv()
if err != nil {
return 0, err
}
updated := time.Unix(job.Service.Updated, 0)
authinfo := job.Service.Oauth
if authinfo == nil {
return 0, fmt.Errorf("skip job: no authinfo")
}
api := anaconda.NewTwitterApi(authinfo.AccessToken, authinfo.AccessTokenSecret)
defer api.Close()
v := url.Values{}
v.Set("screen_name", authinfo.NickName) // goth user.NickName == screen_name
tweets, _ := api.GetUserTimeline(v)
n := 0
for i := len(tweets) - 1; i >= 0; i-- {
tweet := tweets[i]
// skip reply status
if tweet.InReplyToStatusID != 0 {
continue
}
url := "https://twitter.com/" + tweet.User.ScreenName + "/status/" + tweet.IdStr
// deterministic uuid or feed will be polluted
uuid1 := uuid.NewV5(uuid.NamespaceURL, url)
tt, err := tweet.CreatedAtTime()
if err != nil || tt.Before(updated) {
continue
}
from := &pb.Feed{
Id: job.Profile.Id,
Name: job.Profile.Name,
Type: job.Profile.Type,
}
var thumbnails []*pb.Thumbnail
for _, media := range tweet.Entities.Media {
if media.Type != "photo" {
continue
}
url := ""
if media.Media_url_https != "" {
url = media.Media_url_https
} else {
url = media.Media_url
}
thumb := &pb.Thumbnail{
Url: url,
Link: media.Expanded_url,
Width: int32(media.Sizes.Small.W),
Height: int32(media.Sizes.Small.H),
}
thumbnails = append(thumbnails, thumb)
}
body := tweet.Text
tags := ttext.ExtractHashtags(body)
for _, tag := range tags {
new := fmt.Sprintf("<a href=\"https://twitter.com/hashtag/%s\">%s</a>", tag, tag)
body = strings.Replace(body, tag, new, -1)
}
urls := ttext.ExtractURLs(tweet.Text)
for _, url := range urls {
new := fmt.Sprintf("<a href=\"%s\">%s</a>", url, url)
body = strings.Replace(body, url, new, -1)
}
entry := &pb.Entry{
Id: uuid1.String(),
Url: url,
Date: tt.Format(time.RFC3339),
Body: body,
RawBody: tweet.Text,
RawLink: url,
From: from,
// To: []*pb.Feed{from},
Thumbnails: thumbnails,
Via: &pb.Via{
Name: "Twitter",
Url: url,
},
ProfileUuid: job.Profile.Uuid,
}
if err := stream.Send(entry); err != nil {
log.Printf("%v.Send(%v) = %v", stream, entry, err)
return n, err
}
n++
}
return n, nil
}