func SaveRecipes(mongoUri string, recipes []*m.Recipe) error {
session := lib.DBConnect(mongoUri)
defer lib.DBClose(session)
collection := session.DB("").C("Recipe")
totalUpdates := 0
totalInserts := 0
for _, recipe := range recipes {
// make sure the recipe doesn't have _id before upsert to avoid strange bugs stuff
temp := *recipe
temp.Id = ""
info, err := collection.Upsert(bson.M{"article_id": recipe.ArticleId}, temp)
if err != nil {
panic(err)
}
if info.UpsertedId != nil {
recipe.Id = info.UpsertedId.(bson.ObjectId)
totalInserts++
} else {
totalUpdates++
}
}
recipeDebugger.Println(totalUpdates, "recipes updated,", totalInserts, "recipes added")
return nil
}
func processArticle(articleUrl string, session *mgo.Session) bool {
artDebugger.Println("article url ", articleUrl)
processor := a.ParseArticleAtURL(articleUrl, true)
if processor.Err != nil {
artDebugger.Println("Failed to process article: ", processor.Err)
return false
}
var isNew bool
var err error
if globalConfig.MongoUrl != "" {
if session == nil {
session = lib.DBConnect(globalConfig.MongoUrl)
defer lib.DBClose(session)
}
artDebugger.Println("Attempting to save article: ", processor.Article)
isNew, err = processor.Article.Save(session)
if err != nil {
lib.Logger.Println(err)
}
}
artDebugger.Println(processor.Article)
return isNew
}
func LoadAllRecipes(mongoUri string) ([]*m.Recipe, error) {
session := lib.DBConnect(mongoUri)
defer lib.DBClose(session)
collection := session.DB("").C("Recipe")
var result []*m.Recipe
err := collection.Find(nil).All(&result)
return result, err
}
func LoadArticleById(mongoUri string, articleId int) (*m.Article, error) {
session := lib.DBConnect(mongoUri)
defer lib.DBClose(session)
articleCol := session.DB("").C("Article")
var result *m.Article
err := articleCol.Find(bson.M{"article_id": articleId}).One(&result)
return result, err
}
func LoadArticles(mongoUri string) ([]*m.Article, error) {
session := lib.DBConnect(mongoUri)
defer lib.DBClose(session)
articleCol := session.DB("").C("Article")
var result []*m.Article
err := articleCol.Find(nil).All(&result)
return result, err
}
func CheckRecipeURLs(mongoUri string, urls []string) ([]string, error) {
session := lib.DBConnect(mongoUri)
defer lib.DBClose(session)
articleCol := session.DB("").C("Recipe")
var rows []*struct {
Url string `bson:"url"`
}
err := articleCol.Find(bson.M{"url": bson.M{"$in": urls}}).Select(bson.M{"url": 1}).All(&rows)
foundURLs := make([]string, 0, len(rows))
for _, row := range rows {
foundURLs = append(foundURLs, row.Url)
}
return foundURLs, err
}
sections = lib.Sections
} else {
sections = strings.Split(sectionStr, ",")
}
/*if output {
w.Init(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintln(w, "Source\tSection\tHeadline\tURL\tTimestamp")
}*/
feedUrls := formatFeedUrls(sites, sections)
// create one session for all saves bruh
var session *mgo.Session
if globalConfig.MongoUrl != "" {
session = lib.DBConnect(globalConfig.MongoUrl)
defer lib.DBClose(session)
}
newArticles := 0
updatedArticles := 0
var wg sync.WaitGroup
ach := make(chan *a.ArticleUrlsChan)
for _, url := range feedUrls {
go a.GetArticleUrlsFromFeed(url, ach)
aurls := <-ach
for _, aurl := range aurls.Urls {
host, _ := lib.GetHost(url)
wg.Add(1)
go func(url string) {
func RunChartbeatCommands(beats []f.Beat) {
// Set up environment
var session *mgo.Session
if globalConfig.MongoUrl != "" {
session = lib.DBConnect(globalConfig.MongoUrl)
defer lib.DBClose(session)
}
var sites []string
if siteStr == "all" {
sites = lib.Sites
} else {
sites = strings.Split(siteStr, ",")
}
gnapiDomain := "https://api.michigan.com"
if globalConfig.GnapiDomain != "" {
gnapiDomain = globalConfig.GnapiDomain
}
for {
startTime := time.Now()
// Run the actual meat of the program
var beatWait sync.WaitGroup
for _, beat := range beats {
beatWait.Add(1)
go func(beat f.Beat) {
var _copy *mgo.Session
if session != nil {
_copy = session.Copy()
defer _copy.Close()
}
beat.Run(_copy, globalConfig.ChartbeatApiKey, gnapiDomain, sites)
beatWait.Done()
}(beat)
}
beatWait.Wait()
getElapsedTime(&startTime)
sumRes, err := processSummaries()
if err != nil {
chartbeatDebugger.Println("Summarizer failed: ", err)
} else {
chartbeatDebugger.Printf("%v", sumRes)
}
if loop != -1 {
chartbeatDebugger.Printf("Looping! Sleeping for %d seconds...", loop)
time.Sleep(time.Duration(loop) * time.Second)
chartbeatDebugger.Printf("...and now I'm awake!")
if session != nil {
session.Refresh()
}
} else {
break
}
}
}
func TestSaveTimeInterval(t *testing.T) {
t.Skip("No Mongo tests allowed")
mongoUri := os.Getenv("MONGO_URI")
if mongoUri == "" {
t.Fatalf("No MONGO_URI env variable set")
}
session := lib.DBConnect(mongoUri)
defer lib.DBClose(session)
// Save a bunch of articles
numArticles := 20
articles := make([]*m.Article, 0, numArticles)
for i := 0; i < numArticles; i++ {
article := &m.Article{}
article.ArticleId = i + 1
articles = append(articles, article)
}
/*err := a.SaveArticles(mongoUri, articles)
if err != nil {
t.Fatalf("%v", err)
}*/
// Calculate a bunch of the time intervals
topPages := make([]*mc.TopArticle, 0, numArticles)
visits := map[int]int{}
for i := 0; i < numArticles; i++ {
article := &mc.TopArticle{}
articleId := i + 1
numVisits := lib.RandomInt(500)
article.ArticleId = articleId
article.Visits = numVisits
visits[articleId] = numVisits
topPages = append(topPages, article)
}
CalculateTimeInterval(topPages, session)
// Now check the articles saved and make sure they updated the visits
savedArticles := make([]*m.Article, 0, numArticles)
articleCol := session.DB("").C("Article")
articleCol.Find(bson.M{
"article_id": bson.M{
"$gte": 1,
"$lte": 20,
},
}).All(&savedArticles)
if len(savedArticles) != numArticles {
t.Fatalf("Failed to get the right number of articles from the DB")
}
// Verify the visits match up
for _, article := range savedArticles {
articleId := article.ArticleId
if len(article.Visits) != 1 {
t.Fatalf("Should be exactly one visit in the array, there are %d", len(article.Visits))
}
numVisits, ok := visits[articleId]
if !ok {
t.Fatalf("Failed to find visits in map for articleId=%d", articleId)
}
if numVisits != article.Visits[0].Max {
t.Fatalf("Article %d: Expected: %d visits. Actual: %d visits", articleId, numVisits, article.Visits[0].Max)
}
}
}