func ProcessSpreadsLine(dataQueue <-chan []string, facebookUniqueIds map[string]*synced.IntSetSynced,
averageSecondsPerDay map[string]float64, waitGroup *sync.WaitGroup) {
totalSecondsInTimeSlice := make(map[string][]int64)
for line := range dataQueue {
spreadLine := new(SpreadLine)
spreadLine.Id = line[0]
spreadLine.Date = line[1]
decompressLine, err := file_processing.Decompress(line[2])
if err != nil {
log.Fatal("Failed decompression spread line")
}
if err := json.Unmarshal(decompressLine, &spreadLine.Data); err != nil {
fmt.Printf("Failed to unmarshal data")
}
m, _ := regexp.MatchString("google.*", spreadLine.Id)
if m {
for userId, value := range spreadLine.Data {
parsedId, _ := strconv.ParseInt(userId, 10, 64)
if ok := facebookUniqueIds[spreadLine.Date].Visited(parsedId); ok {
for timeslice := 80; timeslice < 92; timeslice++ { //Timeslice 80-92 = 20:00 - 23:00
if _, ok := value[strconv.Itoa(timeslice)]; ok {
if _, ok := totalSecondsInTimeSlice[strconv.Itoa(timeslice)]; !ok {
totalSecondsInTimeSlice[spreadLine.Date] = append(totalSecondsInTimeSlice[spreadLine.Date], 0)
totalSecondsInTimeSlice[spreadLine.Date] = append(totalSecondsInTimeSlice[spreadLine.Date], 0)
}
totalSecondsInTimeSlice[spreadLine.Date][0] += value[strconv.Itoa(timeslice)][0]
totalSecondsInTimeSlice[spreadLine.Date][1]++
}
}
}
}
}
}
for date, values := range totalSecondsInTimeSlice {
averageSecondsPerDay[date] = float64(values[0]) / float64(values[1])
}
defer waitGroup.Done()
}
func ProcessVisitsLine(dataQueue <-chan []string, facebookUniqueIds map[string]*synced.IntSetSynced,
facebookPageViews *synced.MapViewsSynced, waitGroup *sync.WaitGroup) {
for line := range dataQueue {
visitLine := new(VisitsLine)
visitLine.Id = line[0]
visitLine.Date = line[1]
decompressLine, err := file_processing.Decompress(line[2])
if err != nil {
fmt.Println("Failed decompression visit line")
}
if err := json.Unmarshal(decompressLine, &visitLine.Data); err != nil {
fmt.Printf("Failed to unmarshal data")
}
if visitLine.Id == "facebook.com" {
for userId, pageViews := range visitLine.Data {
parsedId, _ := strconv.ParseInt(userId, 10, 64)
if _, ok := facebookUniqueIds[visitLine.Date]; !ok {
facebookUniqueIds[visitLine.Date] = new(synced.IntSetSynced)
facebookUniqueIds[visitLine.Date].Set = make(map[int64]bool)
}
facebookUniqueIds[visitLine.Date].AddToSet(parsedId)
for _, pageView := range pageViews {
if _, ok := facebookPageViews.Map[visitLine.Date]; !ok {
facebookPageViews.Map[visitLine.Date] = 0
}
facebookPageViews.AddToViews(visitLine.Date, pageView[1])
}
}
}
}
defer waitGroup.Done()
}