Beispiel #1
0
func (cdb ComplaintDB) ResetGlobalStats() {
	if err := cdb.DeletAllGlobalStats(); err != nil {
		cdb.Errorf("Reset/DeleteAll fail, %v", err)
		return
	}

	profiles, err := cdb.GetAllProfiles()
	if err != nil {
		return
	}

	// Upon reset (writing a fresh new singleton), we need to generate a key
	rootKey := datastore.NewKey(cdb.Ctx(), kGlobalStatsKind, "foo", 0, nil)
	key := datastore.NewIncompleteKey(cdb.Ctx(), kGlobalStatsKind, rootKey)
	gs := GlobalStats{
		DatastoreKey: key.Encode(),
	}

	// This is too slow to recalculate this way; it runs into the 10m timeout
	//start := date.ArbitraryDatestring2MidnightPdt("2015/08/09", "2006/01/02").Add(-1 * time.Second)
	start := date.ArbitraryDatestring2MidnightPdt("2016/03/15", "2006/01/02").Add(-1 * time.Second)
	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday

	midnights := date.IntermediateMidnights(start, end.Add(time.Minute))
	for _, m := range midnights {
		dayStart, dayEnd := date.WindowForTime(m)

		dc := DailyCount{Datestring: date.Time2Datestring(dayStart)}

		for _, p := range profiles {
			if keys, err := cdb.GetComplaintKeysInSpanByEmailAddress(dayStart, dayEnd, p.EmailAddress); err != nil {
				cdb.Errorf("Reset/Lookup fail, %v", err)
			} else if len(keys) > 0 {
				dc.NumComplaints += len(keys)
				dc.NumComplainers += 1
			}
		}
		gs.Counts = append(gs.Counts, dc)
	}

	if err := cdb.SaveGlobalStats(gs); err != nil {
		cdb.Errorf("Reset/Save fail, %v", err)
	}
}
Beispiel #2
0
func (cdb ComplaintDB) ResetGlobalStats() {
	if err := cdb.DeletAllGlobalStats(); err != nil {
		cdb.C.Errorf("Reset/DeleteAll fail, %v", err)
		return
	}

	profiles, err := cdb.GetAllProfiles()
	if err != nil {
		return
	}

	// Upon reset (writing a fresh new singleton), we need to generate a key
	rootKey := datastore.NewKey(cdb.C, kGlobalStatsKind, "foo", 0, nil)
	key := datastore.NewIncompleteKey(cdb.C, kGlobalStatsKind, rootKey)
	gs := GlobalStats{
		DatastoreKey: key.Encode(),
	}

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end.AddDate(0, 0, -100)
	midnights := date.IntermediateMidnights(start, end.Add(time.Minute))
	for _, m := range midnights {
		dayStart, dayEnd := date.WindowForTime(m)

		dc := DailyCount{Datestring: date.Time2Datestring(dayStart)}

		for _, p := range profiles {
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(p.EmailAddress, dayStart, dayEnd); err != nil {
				cdb.C.Errorf("Reset/Lookup fail, %v", err)
			} else if len(comp) > 0 {
				dc.NumComplaints += len(comp)
				dc.NumComplainers += 1
			}
		}
		gs.Counts = append(gs.Counts, dc)
	}

	if err := cdb.SaveGlobalStats(gs); err != nil {
		cdb.C.Errorf("Reset/Save fail, %v", err)
	}
	cdb.C.Infof("-- reset !--")
	//cdb.LoadGlobalStats();
}
Beispiel #3
0
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) {
	k := fmt.Sprintf("%s:dailycounts", email) // The 'counts' is so we can have diff memcache objects
	c := []DailyCount{}

	cdb.Debugf("GDC_001", "GetDailyCounts() starting")

	// 	might return: datastore.ErrNoSuchEntity
	if dcs, err := cdb.fetchDailyCountSingleton(k); err == datastore.ErrNoSuchEntity {
		// Singleton not found; we don't care; treat same as empty singleton.
	} else if err != nil {
		cdb.Errorf("error getting item: %v", err)
		return c, err
	} else {
		c = dcs
	}
	cdb.Debugf("GDC_002", "singleton lookup done (%d entries)", len(c))

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end                        // by default, this will trigger no lookups (start=end means no missing)

	if len(c) > 0 {
		start = date.Datestring2MidnightPdt(c[0].Datestring)
	} else {
		cdb.Debugf("GDC_003", "counts empty ! track down oldest every, to start iteration range")
		if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil {
			cdb.Errorf("error looking up first complaint for %s: %v", email, err)
			return c, err
		} else if complaint != nil {
			// We move a day into the past; the algo below assumes we have data for the day 'start',
			// but in this case we don't; so trick it into generating data for today.
			start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1)
		} else {
			// cdb.Infof("  - lookup first ever, but empty\n")
		}
		cdb.Debugf("GDC_004", "start point found")
	}

	// Right after the first complaint: it set start to "now", but end is still yesterday.
	if start.After(end) {
		return c, nil
	}

	// We add a minute, to ensure that the day that contains 'end' is included
	missing := date.IntermediateMidnights(start, end.Add(time.Minute))
	if len(missing) > 0 {
		for _, m := range missing {
			cdb.Debugf("GDC_005", "looking up a single span")
			dayStart, dayEnd := date.WindowForTime(m)
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil {
				return []DailyCount{}, err
			} else {
				c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false})
			}
		}
		sort.Sort(DailyCountDesc(c))

		// Now push back into datastore+memcache
		if err := cdb.putDailyCountSingleton(k, c); err != nil {
			cdb.Errorf("error storing counts singleton item: %v", err)
		}
	}
	cdb.Debugf("GDC_006", "all done")

	return c, nil
}
Beispiel #4
0
func SendComplaintsWithSpan(c appengine.Context, start, end time.Time) (err error) {
	c.Infof("--- Emails, %s -> %s", start, end)

	blacklist := map[string]bool{}
	for _, e := range blacklistAddrs {
		blacklist[e] = true
	}

	cdb := complaintdb.ComplaintDB{C: c, Memcache: true}
	var cps = []types.ComplainerProfile{}
	cps, err = cdb.GetAllProfiles()
	if err != nil {
		return
	}

	complaints_private, complaints_submitted, no_data, sent_ok, sent_fail := 0, 0, 0, 0, 0
	sent_single_ok, sent_single_fail := 0, 0

	for _, cp := range cps {
		var complaints = []types.Complaint{}
		complaints, err = cdb.GetComplaintsInSpanByEmailAddress(cp.EmailAddress, start, end)

		if err != nil {
			c.Errorf("Could not get complaints [%v->%v] for <%s>: %v", start, end, cp.EmailAddress, err)
			no_data++
			continue
		}
		if len(complaints) == 0 {
			no_data++
			continue
		}

		// Emailing disabled; last run was Fri Oct 9, 4am, with data for Oct 8. BKSV is now live.
		if false {
			if cp.CcSfo == true {
				for _, complaint := range complaints {
					if msg, err := GenerateSingleComplaintEmail(c, cp, complaint); err != nil {
						c.Errorf("Could not generate single email to <%s>: %v", cp.EmailAddress, err)
						sent_single_fail++
						continue
					} else {
						if blacklist[cp.EmailAddress] {
							sent_single_fail++
						} else {
							if err := mail.Send(c, msg); err != nil {
								c.Errorf("Could not send email to <%s>: %v", cp.EmailAddress, err)
								sent_single_fail++
								continue
							} else {
								sent_single_ok++
							}
						}
					}
				}
			}
		}

		var cap = types.ComplaintsAndProfile{
			Profile:    cp,
			Complaints: complaints,
		}

		var msg *mail.Message
		if msg, err = GenerateEmail(c, cap); err != nil {
			c.Errorf("Could not generate email to <%s>: %v", cp.EmailAddress, err)
			sent_fail++
			continue
		}

		useGateway := false

		if useGateway {
			if err = sendViaHTTPGateway(c, msg); err != nil {
				c.Errorf("Could not gateway email to <%s>: %v", cp.EmailAddress, err)
				sent_fail++
				continue
			}
		} else {
			if blacklist[cp.EmailAddress] {
				sent_fail++
			} else {
				if err = mail.Send(c, msg); err != nil {
					c.Errorf("Could not send email to <%s>: %v", cp.EmailAddress, err)
					sent_fail++
					continue
				}
			}
		}

		if cap.Profile.CcSfo == true {
			complaints_submitted += len(cap.Complaints)
		} else {
			complaints_private += len(cap.Complaints)
		}
		sent_ok++
	}

	subject := fmt.Sprintf("Daily report stats: users:%d/%d  reports:%d/%d  emails:%d:%d",
		sent_ok, (sent_ok + no_data),
		complaints_submitted, (complaints_submitted + complaints_private),
		sent_single_ok, sent_single_fail)
	SendEmailToAdmin(c, subject, "")

	dc := complaintdb.DailyCount{
		Datestring:     date.Time2Datestring(start.Add(time.Hour)),
		NumComplaints:  complaints_submitted + complaints_private,
		NumComplainers: sent_ok,
	}
	cdb.AddDailyCount(dc)

	c.Infof("--- email wrapup: %d ok, %d fail (%d no data) : %d reports submitted (%d kept back)  single[%d/%d]",
		sent_ok, sent_fail, no_data, complaints_submitted, complaints_private, sent_single_ok, sent_single_fail)

	return
}
Beispiel #5
0
func SendComplaintsWithSpan(r *http.Request, start, end time.Time) (err error, str string) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)
	cdb.Infof("--- Emails, %s -> %s", start, end)

	blacklist := map[string]bool{}
	for _, e := range blacklistAddrs {
		blacklist[e] = true
	}

	var cps = []types.ComplainerProfile{}
	cps, err = cdb.GetAllProfiles()
	if err != nil {
		return
	}

	complaints_private, complaints_submitted, no_data, sent_ok, sent_fail := 0, 0, 0, 0, 0
	sent_single_ok, sent_single_fail := 0, 0

	for _, cp := range cps {
		var complaints = []types.Complaint{}
		complaints, err = cdb.GetComplaintsInSpanByEmailAddress(cp.EmailAddress, start, end)

		if err != nil {
			cdb.Errorf("Could not get complaints [%v->%v] for <%s>: %v", start, end, cp.EmailAddress, err)
			no_data++
			continue
		}
		if len(complaints) == 0 {
			no_data++
			continue
		}

		var cap = types.ComplaintsAndProfile{
			Profile:    cp,
			Complaints: complaints,
		}

		var msg *mail.Message
		if msg, err = GenerateEmail(ctx, cap); err != nil {
			cdb.Errorf("Could not generate email to <%s>: %v", cp.EmailAddress, err)
			sent_fail++
			continue
		}

		if blacklist[cp.EmailAddress] {
			sent_fail++
		} else {
			if err = mail.Send(ctx, msg); err != nil {
				cdb.Errorf("Could not send email to <%s>: %v", cp.EmailAddress, err)
				sent_fail++
				continue
			}
		}

		complaints_submitted += len(cap.Complaints)
		sent_ok++
	}

	subject := fmt.Sprintf("Daily report stats: users:%d/%d  reports:%d/%d  emails:%d:%d",
		sent_ok, (sent_ok + no_data),
		complaints_submitted, (complaints_submitted + complaints_private),
		sent_single_ok, sent_single_fail)

	SendEmailToAdmin(ctx, subject, "")

	dc := complaintdb.DailyCount{
		Datestring:     date.Time2Datestring(start.Add(time.Hour)),
		NumComplaints:  complaints_submitted + complaints_private,
		NumComplainers: sent_ok,
	}
	cdb.AddDailyCount(dc)

	str = fmt.Sprintf("email wrapup: %d ok, %d fail (%d no data) : %d reports submitted (%d kept back)  single[%d/%d]", sent_ok, sent_fail, no_data, complaints_submitted, complaints_private, sent_single_ok, sent_single_fail)

	cdb.Infof("--- %s", str)

	return
}
Beispiel #6
0
// http://stackoverflow.com/questions/13264555/store-an-object-in-memcache-of-gae-in-go
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) {
	// cdb.C.Infof("-- GetDaily for %s", email)

	k := fmt.Sprintf("%s:daily", email)
	c := []DailyCount{}

	if _, err := memcache.Gob.Get(cdb.C, k, &c); err == memcache.ErrCacheMiss {
		// cache miss, but we don't care
	} else if err != nil {
		cdb.C.Errorf("error getting item: %v", err)
		return c, err
	}

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end                        // by default, this will trigger no lookups (start=end means no missing)

	if len(c) > 0 {
		start = date.Datestring2MidnightPdt(c[0].Datestring)
	} else {
		if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil {
			cdb.C.Errorf("error looking up first complaint for %s: %v", email, err)
			return c, err
		} else if complaint != nil {
			// We move a day into the past; the algo below assumes we have data for the day 'start',
			// but in this case we don't; so trick it into generating data for today.
			start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1)
			//cdb.C.Infof("  - lookup first ever, %s", complaint.Timestamp)
		} else {
			// cdb.C.Infof("  - lookup first ever, but empty\n")
		}
	}

	// Right after the first complaint: it set start to "now", but end is still yesterday.
	if start.After(end) {
		// cdb.C.Infof("--- s>e {%s} > {%s}\n", start, end)
		return c, nil
	}

	// We add a minute, to ensure that the day that contains 'end' is included
	missing := date.IntermediateMidnights(start, end.Add(time.Minute))
	// cdb.C.Infof("--- missing? --- {%s} -> {%s} == %d\n", start, end.Add(time.Minute), len(missing))
	if len(missing) > 0 {
		for _, m := range missing {
			dayStart, dayEnd := date.WindowForTime(m)
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil {
				return []DailyCount{}, err
			} else {
				// cdb.C.Infof("  -  {%s}  n=%d [%v]\n", dayStart, len(comp), m)
				c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false})
			}
		}
		sort.Sort(DailyCountDesc(c))

		// Now push back into memcache
		item := memcache.Item{Key: k, Object: c}
		if err := memcache.Gob.Set(cdb.C, &item); err != nil {
			cdb.C.Errorf("error setting item: %v", err)
		}
	}

	// cdb.C.Infof("--- done")
	return c, nil
}