예제 #1
0
func (a *ArticleIn) GetData() error {

	artDebugger.Println("Fetching: ", a.Url)

	doc, err := gq.NewDocument(a.Url)
	if err != nil {
		return err
	}

	a.Site, _ = lib.GetHost(a.Url)
	a.Doc = doc
	a.ArticleId = lib.GetArticleId(a.Url)

	return nil
}
예제 #2
0
		var session *mgo.Session
		if globalConfig.MongoUrl != "" {
			session = lib.DBConnect(globalConfig.MongoUrl)
			defer lib.DBClose(session)
		}

		newArticles := 0
		updatedArticles := 0

		var wg sync.WaitGroup
		ach := make(chan *a.ArticleUrlsChan)
		for _, url := range feedUrls {
			go a.GetArticleUrlsFromFeed(url, ach)
			aurls := <-ach
			for _, aurl := range aurls.Urls {
				host, _ := lib.GetHost(url)
				wg.Add(1)
				go func(url string) {
					defer wg.Done()
					// http://stackoverflow.com/questions/26574594/best-practice-to-maintain-a-mgo-session
					var gosesh *mgo.Session
					if session != nil {
						gosesh := session.Copy()
						defer gosesh.Close()
					}

					isNew := processArticle(url, gosesh)
					if isNew {
						newArticles++
					} else {
						updatedArticles++