func SaveRecipes(mongoUri string, recipes []*m.Recipe) error {
	session := lib.DBConnect(mongoUri)
	defer lib.DBClose(session)

	collection := session.DB("").C("Recipe")

	totalUpdates := 0
	totalInserts := 0
	for _, recipe := range recipes {
		// make sure the recipe doesn't have _id before upsert to avoid strange bugs stuff
		temp := *recipe
		temp.Id = ""
		info, err := collection.Upsert(bson.M{"article_id": recipe.ArticleId}, temp)
		if err != nil {
			panic(err)
		}

		if info.UpsertedId != nil {
			recipe.Id = info.UpsertedId.(bson.ObjectId)
			totalInserts++
		} else {
			totalUpdates++
		}
	}
	recipeDebugger.Println(totalUpdates, "recipes updated,", totalInserts, "recipes added")

	return nil
}
Beispiel #2
0
func processArticle(articleUrl string, session *mgo.Session) bool {
	artDebugger.Println("article url ", articleUrl)
	processor := a.ParseArticleAtURL(articleUrl, true)
	if processor.Err != nil {
		artDebugger.Println("Failed to process article: ", processor.Err)
		return false
	}

	var isNew bool
	var err error
	if globalConfig.MongoUrl != "" {
		if session == nil {
			session = lib.DBConnect(globalConfig.MongoUrl)
			defer lib.DBClose(session)
		}

		artDebugger.Println("Attempting to save article: ", processor.Article)
		isNew, err = processor.Article.Save(session)
		if err != nil {
			lib.Logger.Println(err)
		}
	}

	artDebugger.Println(processor.Article)
	return isNew
}
func LoadAllRecipes(mongoUri string) ([]*m.Recipe, error) {
	session := lib.DBConnect(mongoUri)
	defer lib.DBClose(session)

	collection := session.DB("").C("Recipe")
	var result []*m.Recipe
	err := collection.Find(nil).All(&result)
	return result, err
}
Beispiel #4
0
func LoadArticleById(mongoUri string, articleId int) (*m.Article, error) {
	session := lib.DBConnect(mongoUri)
	defer lib.DBClose(session)

	articleCol := session.DB("").C("Article")

	var result *m.Article
	err := articleCol.Find(bson.M{"article_id": articleId}).One(&result)
	return result, err
}
Beispiel #5
0
func LoadArticles(mongoUri string) ([]*m.Article, error) {
	session := lib.DBConnect(mongoUri)
	defer lib.DBClose(session)

	articleCol := session.DB("").C("Article")

	var result []*m.Article
	err := articleCol.Find(nil).All(&result)
	return result, err
}
Beispiel #6
0
func CheckRecipeURLs(mongoUri string, urls []string) ([]string, error) {
	session := lib.DBConnect(mongoUri)
	defer lib.DBClose(session)

	articleCol := session.DB("").C("Recipe")

	var rows []*struct {
		Url string `bson:"url"`
	}
	err := articleCol.Find(bson.M{"url": bson.M{"$in": urls}}).Select(bson.M{"url": 1}).All(&rows)

	foundURLs := make([]string, 0, len(rows))
	for _, row := range rows {
		foundURLs = append(foundURLs, row.Url)
	}

	return foundURLs, err
}
Beispiel #7
0
		} else {
			sections = strings.Split(sectionStr, ",")
		}

		/*if output {
			w.Init(os.Stdout, 0, 8, 0, '\t', 0)
			fmt.Fprintln(w, "Source\tSection\tHeadline\tURL\tTimestamp")
		}*/

		feedUrls := formatFeedUrls(sites, sections)

		// create one session for all saves bruh
		var session *mgo.Session
		if globalConfig.MongoUrl != "" {
			session = lib.DBConnect(globalConfig.MongoUrl)
			defer lib.DBClose(session)
		}

		newArticles := 0
		updatedArticles := 0

		var wg sync.WaitGroup
		ach := make(chan *a.ArticleUrlsChan)
		for _, url := range feedUrls {
			go a.GetArticleUrlsFromFeed(url, ach)
			aurls := <-ach
			for _, aurl := range aurls.Urls {
				host, _ := lib.GetHost(url)
				wg.Add(1)
				go func(url string) {
					defer wg.Done()
Beispiel #8
0
func RunChartbeatCommands(beats []f.Beat) {
	// Set up environment
	var session *mgo.Session
	if globalConfig.MongoUrl != "" {
		session = lib.DBConnect(globalConfig.MongoUrl)
		defer lib.DBClose(session)
	}

	var sites []string

	if siteStr == "all" {
		sites = lib.Sites
	} else {
		sites = strings.Split(siteStr, ",")
	}

	gnapiDomain := "https://api.michigan.com"
	if globalConfig.GnapiDomain != "" {
		gnapiDomain = globalConfig.GnapiDomain
	}

	for {
		startTime := time.Now()

		// Run the actual meat of the program
		var beatWait sync.WaitGroup

		for _, beat := range beats {
			beatWait.Add(1)

			go func(beat f.Beat) {
				var _copy *mgo.Session
				if session != nil {
					_copy = session.Copy()
					defer _copy.Close()
				}
				beat.Run(_copy, globalConfig.ChartbeatApiKey, gnapiDomain, sites)
				beatWait.Done()
			}(beat)
		}

		beatWait.Wait()

		getElapsedTime(&startTime)

		sumRes, err := processSummaries()
		if err != nil {
			chartbeatDebugger.Println("Summarizer failed: ", err)
		} else {
			chartbeatDebugger.Printf("%v", sumRes)
		}

		if loop != -1 {
			chartbeatDebugger.Printf("Looping! Sleeping for %d seconds...", loop)
			time.Sleep(time.Duration(loop) * time.Second)
			chartbeatDebugger.Printf("...and now I'm awake!")
			if session != nil {
				session.Refresh()
			}
		} else {
			break
		}
	}
}
func TestSaveTimeInterval(t *testing.T) {
	t.Skip("No Mongo tests allowed")
	mongoUri := os.Getenv("MONGO_URI")
	if mongoUri == "" {
		t.Fatalf("No MONGO_URI env variable set")
	}

	session := lib.DBConnect(mongoUri)
	defer lib.DBClose(session)

	// Save a bunch of articles
	numArticles := 20
	articles := make([]*m.Article, 0, numArticles)
	for i := 0; i < numArticles; i++ {
		article := &m.Article{}
		article.ArticleId = i + 1
		articles = append(articles, article)
	}
	/*err := a.SaveArticles(mongoUri, articles)
	if err != nil {
		t.Fatalf("%v", err)
	}*/

	// Calculate a bunch of the time intervals
	topPages := make([]*mc.TopArticle, 0, numArticles)
	visits := map[int]int{}
	for i := 0; i < numArticles; i++ {
		article := &mc.TopArticle{}
		articleId := i + 1
		numVisits := lib.RandomInt(500)

		article.ArticleId = articleId
		article.Visits = numVisits

		visits[articleId] = numVisits
		topPages = append(topPages, article)
	}
	CalculateTimeInterval(topPages, session)

	// Now check the articles saved and make sure they updated the visits
	savedArticles := make([]*m.Article, 0, numArticles)
	articleCol := session.DB("").C("Article")
	articleCol.Find(bson.M{
		"article_id": bson.M{
			"$gte": 1,
			"$lte": 20,
		},
	}).All(&savedArticles)

	if len(savedArticles) != numArticles {
		t.Fatalf("Failed to get the right number of articles from the DB")
	}

	// Verify the visits match up
	for _, article := range savedArticles {
		articleId := article.ArticleId
		if len(article.Visits) != 1 {
			t.Fatalf("Should be exactly one visit in the array, there are %d", len(article.Visits))
		}

		numVisits, ok := visits[articleId]
		if !ok {
			t.Fatalf("Failed to find visits in map for articleId=%d", articleId)
		}

		if numVisits != article.Visits[0].Max {
			t.Fatalf("Article %d: Expected: %d visits. Actual: %d visits", articleId, numVisits, article.Visits[0].Max)
		}
	}
}