Exemple #1
0
func perftesterHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	tStart := time.Now()
	str := ""
	debugf := func(step string, fmtstr string, varargs ...interface{}) {
		payload := fmt.Sprintf(fmtstr, varargs...)
		outStr := fmt.Sprintf("[%s] %9.6f %s", step, time.Since(tStart).Seconds(), payload)
		log.Debugf(ctx, outStr)
		str += "* " + outStr + "\n"
	}

	debugf("pt_001", "starting")

	stats := map[string]*complaintdb.DailyCount{}
	dailys := []complaintdb.DailyCount{}
	counts := []types.CountItem{}

	debugf("pt_002", "populating")
	t := time.Now().AddDate(0, -6, 0)
	for i := 0; i < 150; i++ {
		t = t.AddDate(0, 0, 1)
		dailys = append(dailys, complaintdb.DailyCount{t.Format("2006.01.02"), i, i, false, false})
		item := complaintdb.DailyCount{t.Format("2006.01.02"), i, i, false, false}
		stats[date.Datestring2MidnightPdt(item.Datestring).Format("Jan 02")] = &item
	}
	debugf("pt_005", "populating all done")

	pdt, _ := time.LoadLocation("America/Los_Angeles")
	dateNoTimeFormat := "2006.01.02"
	arbitraryDatestring2MidnightPdt := func(s string, fmt string) time.Time {
		t, _ := time.ParseInLocation(fmt, s, pdt)
		return t
	}
	datestring2MidnightPdt := func(s string) time.Time {
		return arbitraryDatestring2MidnightPdt(s, dateNoTimeFormat)
	}
	_ = datestring2MidnightPdt

	debugf("pt_010", "daily stats loaded (%d dailys, %d stats)", len(dailys), len(stats))
	for _, daily := range dailys {
		// cdb.C.Infof(" -- we have a daily: %#v", daily)
		//key := datestring2MidnightPdt(daily.Datestring).Format("Jan 02")
		item := types.CountItem{
			//Key: key, //fmt.Sprintf("Jan %02d", j+1), //daily.Timestamp().Format("Jan 02"),
			Key:   daily.Timestamp().Format("Jan 02"),
			Count: daily.NumComplaints,
		}
		if dc, exists := stats[item.Key]; exists {
			item.TotalComplainers = dc.NumComplainers
			item.TotalComplaints = dc.NumComplaints
			item.IsMaxComplainers = dc.IsMaxComplainers
			item.IsMaxComplaints = dc.IsMaxComplaints
		}
		//counts = append(counts, item)
	}
	debugf("pt_090", "daily stats munged (%d counts)", len(counts))

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(str))
}
Exemple #2
0
func (cdb ComplaintDB) getDailyCountsByEmailAdress(ea string) ([]types.CountItem, error) {
	counts := []types.CountItem{}

	gs, _ := cdb.LoadGlobalStats()
	stats := map[string]*DailyCount{}
	if gs != nil {
		for i, dc := range gs.Counts {
			stats[date.Datestring2MidnightPdt(dc.Datestring).Format("Jan 02")] = &gs.Counts[i]
		}
	}

	if dailys, err := cdb.GetDailyCounts(ea); err != nil {
		return counts, err

	} else {
		for _, daily := range dailys {
			// cdb.C.Infof(" -- we have a daily: %#v", daily)
			item := types.CountItem{
				Key:   daily.Timestamp().Format("Jan 02"),
				Count: daily.NumComplaints,
			}
			if dc, exists := stats[item.Key]; exists {
				item.TotalComplainers = dc.NumComplainers
				item.TotalComplaints = dc.NumComplaints
				item.IsMaxComplainers = dc.IsMaxComplainers
				item.IsMaxComplaints = dc.IsMaxComplaints
			}
			counts = append(counts, item)
		}
	}

	return counts, nil
}
Exemple #3
0
func (cdb ComplaintDB) getDailyCountsByEmailAdress(ea string) ([]types.CountItem, error) {
	cdb.Debugf("gDCBEA_001", "starting")
	gs, _ := cdb.LoadGlobalStats()
	cdb.Debugf("gDCBEA_002", "global stats loaded")
	stats := map[string]*DailyCount{}
	if gs != nil {
		for i, dc := range gs.Counts {
			stats[date.Datestring2MidnightPdt(dc.Datestring).Format("Jan 02")] = &gs.Counts[i]
		}
	}
	cdb.Debugf("gDCBEA_003", "global stats munged; loading daily")

	dailys, err := cdb.GetDailyCounts(ea)
	if err != nil {
		return []types.CountItem{}, err
	}

	counts := []types.CountItem{}

	cdb.Debugf("gDCBEA_004", "daily stats loaded (%d dailys, %d stats)", len(dailys), len(stats))
	for _, daily := range dailys {
		item := types.CountItem{
			Key:   daily.Timestamp().Format("Jan 02"),
			Count: daily.NumComplaints,
		}
		if dc, exists := stats[item.Key]; exists {
			item.TotalComplainers = dc.NumComplainers
			item.TotalComplaints = dc.NumComplaints
			item.IsMaxComplainers = dc.IsMaxComplainers
			item.IsMaxComplaints = dc.IsMaxComplaints
		}
		counts = append(counts, item)
	}
	cdb.Debugf("gDCBEA_005", "daily stats munged (%d counts)", len(counts))

	return counts, nil
}
Exemple #4
0
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) {
	k := fmt.Sprintf("%s:dailycounts", email) // The 'counts' is so we can have diff memcache objects
	c := []DailyCount{}

	cdb.Debugf("GDC_001", "GetDailyCounts() starting")

	// 	might return: datastore.ErrNoSuchEntity
	if dcs, err := cdb.fetchDailyCountSingleton(k); err == datastore.ErrNoSuchEntity {
		// Singleton not found; we don't care; treat same as empty singleton.
	} else if err != nil {
		cdb.Errorf("error getting item: %v", err)
		return c, err
	} else {
		c = dcs
	}
	cdb.Debugf("GDC_002", "singleton lookup done (%d entries)", len(c))

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end                        // by default, this will trigger no lookups (start=end means no missing)

	if len(c) > 0 {
		start = date.Datestring2MidnightPdt(c[0].Datestring)
	} else {
		cdb.Debugf("GDC_003", "counts empty ! track down oldest every, to start iteration range")
		if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil {
			cdb.Errorf("error looking up first complaint for %s: %v", email, err)
			return c, err
		} else if complaint != nil {
			// We move a day into the past; the algo below assumes we have data for the day 'start',
			// but in this case we don't; so trick it into generating data for today.
			start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1)
		} else {
			// cdb.Infof("  - lookup first ever, but empty\n")
		}
		cdb.Debugf("GDC_004", "start point found")
	}

	// Right after the first complaint: it set start to "now", but end is still yesterday.
	if start.After(end) {
		return c, nil
	}

	// We add a minute, to ensure that the day that contains 'end' is included
	missing := date.IntermediateMidnights(start, end.Add(time.Minute))
	if len(missing) > 0 {
		for _, m := range missing {
			cdb.Debugf("GDC_005", "looking up a single span")
			dayStart, dayEnd := date.WindowForTime(m)
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil {
				return []DailyCount{}, err
			} else {
				c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false})
			}
		}
		sort.Sort(DailyCountDesc(c))

		// Now push back into datastore+memcache
		if err := cdb.putDailyCountSingleton(k, c); err != nil {
			cdb.Errorf("error storing counts singleton item: %v", err)
		}
	}
	cdb.Debugf("GDC_006", "all done")

	return c, nil
}
Exemple #5
0
func (dc DailyCount) Timestamp() time.Time {
	return date.Datestring2MidnightPdt(dc.Datestring)
}
Exemple #6
0
// Returns number of records written (which is zero if the file already exists)
func writeAnonymizedGCSFile(r *http.Request, datestring, foldername, filename string) (int, error) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)

	// Get a list of users that as of right now, have opted out of data sharing.
	optOutUsers, err := cdb.GetComplainersCurrentlyOptedOut()
	if err != nil {
		return 0, fmt.Errorf("get optout users: %v", err)
	}

	if exists, err := gcs.Exists(ctx, foldername, filename); err != nil {
		return 0, err
	} else if exists {
		return 0, nil
	}

	gcsHandle, err := gcs.OpenRW(ctx, foldername, filename, "application/json")
	if err != nil {
		return 0, err
	}

	encoder := json.NewEncoder(gcsHandle.IOWriter())

	s := date.Datestring2MidnightPdt(datestring)
	e := s.AddDate(0, 0, 1).Add(-1 * time.Second) // +23:59:59 (or 22:59 or 24:59 when going in/out DST)

	n := 0
	// An iterator expires after 60s, no matter what; so carve up into short-lived iterators
	for _, dayWindow := range DayWindows(s, e) {
		iter := cdb.NewLongBatchingIter(cdb.QueryInSpan(dayWindow[0], dayWindow[1]))

		for {
			c, err := iter.NextWithErr()
			if err != nil {
				return 0, fmt.Errorf("iterator [%s,%s] failed at %s: %v",
					dayWindow[0], dayWindow[1], time.Now(), err)
			} else if c == nil {
				break // we're all done with this iterator
			}

			// If the user is currently opted out, ignore their data
			if _, exists := optOutUsers[c.Profile.EmailAddress]; exists {
				continue
			}

			n++
			ac := complaintdb.AnonymizeComplaint(c)

			if err := encoder.Encode(ac); err != nil {
				return 0, err
			}
		}
	}

	if err := gcsHandle.Close(); err != nil {
		return 0, err
	}

	log.Infof(ctx, "GCS bigquery file '%s' successfully written", filename)

	return n, nil
}
Exemple #7
0
// http://stackoverflow.com/questions/13264555/store-an-object-in-memcache-of-gae-in-go
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) {
	// cdb.C.Infof("-- GetDaily for %s", email)

	k := fmt.Sprintf("%s:daily", email)
	c := []DailyCount{}

	if _, err := memcache.Gob.Get(cdb.C, k, &c); err == memcache.ErrCacheMiss {
		// cache miss, but we don't care
	} else if err != nil {
		cdb.C.Errorf("error getting item: %v", err)
		return c, err
	}

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end                        // by default, this will trigger no lookups (start=end means no missing)

	if len(c) > 0 {
		start = date.Datestring2MidnightPdt(c[0].Datestring)
	} else {
		if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil {
			cdb.C.Errorf("error looking up first complaint for %s: %v", email, err)
			return c, err
		} else if complaint != nil {
			// We move a day into the past; the algo below assumes we have data for the day 'start',
			// but in this case we don't; so trick it into generating data for today.
			start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1)
			//cdb.C.Infof("  - lookup first ever, %s", complaint.Timestamp)
		} else {
			// cdb.C.Infof("  - lookup first ever, but empty\n")
		}
	}

	// Right after the first complaint: it set start to "now", but end is still yesterday.
	if start.After(end) {
		// cdb.C.Infof("--- s>e {%s} > {%s}\n", start, end)
		return c, nil
	}

	// We add a minute, to ensure that the day that contains 'end' is included
	missing := date.IntermediateMidnights(start, end.Add(time.Minute))
	// cdb.C.Infof("--- missing? --- {%s} -> {%s} == %d\n", start, end.Add(time.Minute), len(missing))
	if len(missing) > 0 {
		for _, m := range missing {
			dayStart, dayEnd := date.WindowForTime(m)
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil {
				return []DailyCount{}, err
			} else {
				// cdb.C.Infof("  -  {%s}  n=%d [%v]\n", dayStart, len(comp), m)
				c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false})
			}
		}
		sort.Sort(DailyCountDesc(c))

		// Now push back into memcache
		item := memcache.Item{Key: k, Object: c}
		if err := memcache.Gob.Set(cdb.C, &item); err != nil {
			cdb.C.Errorf("error setting item: %v", err)
		}
	}

	// cdb.C.Infof("--- done")
	return c, nil
}