Пример #1
0
func ProcessSpreadsLine(dataQueue <-chan []string, facebookUniqueIds map[string]*synced.IntSetSynced,
	averageSecondsPerDay map[string]float64, waitGroup *sync.WaitGroup) {
	totalSecondsInTimeSlice := make(map[string][]int64)
	for line := range dataQueue {
		spreadLine := new(SpreadLine)
		spreadLine.Id = line[0]
		spreadLine.Date = line[1]

		decompressLine, err := file_processing.Decompress(line[2])

		if err != nil {
			log.Fatal("Failed decompression spread line")
		}

		if err := json.Unmarshal(decompressLine, &spreadLine.Data); err != nil {
			fmt.Printf("Failed to unmarshal data")
		}

		m, _ := regexp.MatchString("google.*", spreadLine.Id)
		if m {
			for userId, value := range spreadLine.Data {
				parsedId, _ := strconv.ParseInt(userId, 10, 64)
				if ok := facebookUniqueIds[spreadLine.Date].Visited(parsedId); ok {
					for timeslice := 80; timeslice < 92; timeslice++ { //Timeslice 80-92 = 20:00 - 23:00
						if _, ok := value[strconv.Itoa(timeslice)]; ok {
							if _, ok := totalSecondsInTimeSlice[strconv.Itoa(timeslice)]; !ok {
								totalSecondsInTimeSlice[spreadLine.Date] = append(totalSecondsInTimeSlice[spreadLine.Date], 0)
								totalSecondsInTimeSlice[spreadLine.Date] = append(totalSecondsInTimeSlice[spreadLine.Date], 0)
							}
							totalSecondsInTimeSlice[spreadLine.Date][0] += value[strconv.Itoa(timeslice)][0]
							totalSecondsInTimeSlice[spreadLine.Date][1]++
						}
					}
				}
			}
		}
	}
	for date, values := range totalSecondsInTimeSlice {
		averageSecondsPerDay[date] = float64(values[0]) / float64(values[1])
	}
	defer waitGroup.Done()
}
Пример #2
0
func ProcessVisitsLine(dataQueue <-chan []string, facebookUniqueIds map[string]*synced.IntSetSynced,
	facebookPageViews *synced.MapViewsSynced, waitGroup *sync.WaitGroup) {
	for line := range dataQueue {
		visitLine := new(VisitsLine)
		visitLine.Id = line[0]
		visitLine.Date = line[1]
		decompressLine, err := file_processing.Decompress(line[2])

		if err != nil {
			fmt.Println("Failed decompression visit line")
		}

		if err := json.Unmarshal(decompressLine, &visitLine.Data); err != nil {
			fmt.Printf("Failed to unmarshal data")
		}

		if visitLine.Id == "facebook.com" {
			for userId, pageViews := range visitLine.Data {
				parsedId, _ := strconv.ParseInt(userId, 10, 64)
				if _, ok := facebookUniqueIds[visitLine.Date]; !ok {
					facebookUniqueIds[visitLine.Date] = new(synced.IntSetSynced)
					facebookUniqueIds[visitLine.Date].Set = make(map[int64]bool)
				}

				facebookUniqueIds[visitLine.Date].AddToSet(parsedId)
				for _, pageView := range pageViews {
					if _, ok := facebookPageViews.Map[visitLine.Date]; !ok {
						facebookPageViews.Map[visitLine.Date] = 0
					}
					facebookPageViews.AddToViews(visitLine.Date, pageView[1])
				}
			}
		}
	}
	defer waitGroup.Done()
}