func (a *ArticleIn) GetData() error {
artDebugger.Println("Fetching: ", a.Url)
doc, err := gq.NewDocument(a.Url)
if err != nil {
return err
}
a.Site, _ = lib.GetHost(a.Url)
a.Doc = doc
a.ArticleId = lib.GetArticleId(a.Url)
return nil
}
var session *mgo.Session
if globalConfig.MongoUrl != "" {
session = lib.DBConnect(globalConfig.MongoUrl)
defer lib.DBClose(session)
}
newArticles := 0
updatedArticles := 0
var wg sync.WaitGroup
ach := make(chan *a.ArticleUrlsChan)
for _, url := range feedUrls {
go a.GetArticleUrlsFromFeed(url, ach)
aurls := <-ach
for _, aurl := range aurls.Urls {
host, _ := lib.GetHost(url)
wg.Add(1)
go func(url string) {
defer wg.Done()
// http://stackoverflow.com/questions/26574594/best-practice-to-maintain-a-mgo-session
var gosesh *mgo.Session
if session != nil {
gosesh := session.Copy()
defer gosesh.Close()
}
isNew := processArticle(url, gosesh)
if isNew {
newArticles++
} else {
updatedArticles++