func cannedSerfr1ComplaintsHandler(w http.ResponseWriter, r *http.Request) { s, e := date.WindowForYesterday() e = e.Add(-1 * time.Second) format := "list" if r.FormValue("csv") != "" { format = "csv" } c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout reportWriter(c, w, r, s, e, ReportOptions{}, "serfr1complaints", format) }
func cannedDiscrepHandler(w http.ResponseWriter, r *http.Request) { s, e := date.WindowForYesterday() e = e.Add(-1 * time.Second) opt := ReportOptions{} format := "list" if r.FormValue("csv") != "" { format = "csv" } reportWriter(appengine.NewContext(r), w, r, s, e, opt, "discrep", format) }
func cannedAdsbClassBHandler(w http.ResponseWriter, r *http.Request) { s, e := date.WindowForYesterday() e = e.Add(-1 * time.Second) opt := ReportOptions{} format := "list" if r.FormValue("csv") != "" { format = "csv" } c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout reportWriter(c, w, r, s, e, opt, "adsbclassb", format) }
func debugHandler3(w http.ResponseWriter, r *http.Request) { ctx := req2ctx(r) cdb := complaintdb.NewDB(ctx) tStart := time.Now() start, end := date.WindowForYesterday() keys, err := cdb.GetComplaintKeysInSpan(start, end) str := fmt.Sprintf("OK\nret: %d\nerr: %v\nElapsed: %s\ns: %s\ne: %s\n", len(keys), err, time.Since(tStart), start, end) w.Header().Set("Content-Type", "text/plain") w.Write([]byte(str)) }
func cannedSerfr1Handler(w http.ResponseWriter, r *http.Request) { s, e := date.WindowForYesterday() e = e.Add(-1 * time.Second) opt := ReportOptions{ ClassB_OnePerFlight: true, } format := "list" if r.FormValue("csv") != "" { format = "csv" } reportWriter(appengine.NewContext(r), w, r, s, e, opt, "serfr1", format) }
func (cdb ComplaintDB) ResetGlobalStats() { if err := cdb.DeletAllGlobalStats(); err != nil { cdb.Errorf("Reset/DeleteAll fail, %v", err) return } profiles, err := cdb.GetAllProfiles() if err != nil { return } // Upon reset (writing a fresh new singleton), we need to generate a key rootKey := datastore.NewKey(cdb.Ctx(), kGlobalStatsKind, "foo", 0, nil) key := datastore.NewIncompleteKey(cdb.Ctx(), kGlobalStatsKind, rootKey) gs := GlobalStats{ DatastoreKey: key.Encode(), } // This is too slow to recalculate this way; it runs into the 10m timeout //start := date.ArbitraryDatestring2MidnightPdt("2015/08/09", "2006/01/02").Add(-1 * time.Second) start := date.ArbitraryDatestring2MidnightPdt("2016/03/15", "2006/01/02").Add(-1 * time.Second) end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday midnights := date.IntermediateMidnights(start, end.Add(time.Minute)) for _, m := range midnights { dayStart, dayEnd := date.WindowForTime(m) dc := DailyCount{Datestring: date.Time2Datestring(dayStart)} for _, p := range profiles { if keys, err := cdb.GetComplaintKeysInSpanByEmailAddress(dayStart, dayEnd, p.EmailAddress); err != nil { cdb.Errorf("Reset/Lookup fail, %v", err) } else if len(keys) > 0 { dc.NumComplaints += len(keys) dc.NumComplainers += 1 } } gs.Counts = append(gs.Counts, dc) } if err := cdb.SaveGlobalStats(gs); err != nil { cdb.Errorf("Reset/Save fail, %v", err) } }
func bksvSubmitUserHandler(w http.ResponseWriter, r *http.Request) { ctx := req2ctx(r) cdb := complaintdb.NewDB(ctx) client := cdb.HTTPClient() start, end := date.WindowForYesterday() bksv_ok, bksv_not_ok := 0, 0 email := r.FormValue("user") if cp, err := cdb.GetProfileByEmailAddress(email); err != nil { cdb.Errorf(" /bksv/submit-user(%s): getprofile: %v", email, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } else if complaints, err := cdb.GetComplaintsInSpanByEmailAddress(email, start, end); err != nil { cdb.Errorf(" /bksv/submit-user(%s): getcomplaints: %v", email, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } else { for i, complaint := range complaints { time.Sleep(time.Millisecond * 200) if debug, err := bksv.PostComplaint(client, *cp, complaint); err != nil { //cdb.Infof("pro: %v", cp) //cdb.Infof("comp: %#v", complaint) cdb.Errorf("BKSV posting error: %v", err) cdb.Infof("BKSV Debug\n------\n%s\n------\n", debug) bksv_not_ok++ } else { if i == 0 { cdb.Infof("BKSV [OK] Debug\n------\n%s\n------\n", debug) } bksv_ok++ } } } cdb.Infof("bksv for %s, %d/%d", email, bksv_ok, bksv_not_ok) if bksv_not_ok > 0 { cdb.Errorf("bksv for %s, %d/%d", email, bksv_ok, bksv_not_ok) } w.Write([]byte("OK")) }
func (cdb ComplaintDB) ResetGlobalStats() { if err := cdb.DeletAllGlobalStats(); err != nil { cdb.C.Errorf("Reset/DeleteAll fail, %v", err) return } profiles, err := cdb.GetAllProfiles() if err != nil { return } // Upon reset (writing a fresh new singleton), we need to generate a key rootKey := datastore.NewKey(cdb.C, kGlobalStatsKind, "foo", 0, nil) key := datastore.NewIncompleteKey(cdb.C, kGlobalStatsKind, rootKey) gs := GlobalStats{ DatastoreKey: key.Encode(), } end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday start := end.AddDate(0, 0, -100) midnights := date.IntermediateMidnights(start, end.Add(time.Minute)) for _, m := range midnights { dayStart, dayEnd := date.WindowForTime(m) dc := DailyCount{Datestring: date.Time2Datestring(dayStart)} for _, p := range profiles { if comp, err := cdb.GetComplaintsInSpanByEmailAddress(p.EmailAddress, dayStart, dayEnd); err != nil { cdb.C.Errorf("Reset/Lookup fail, %v", err) } else if len(comp) > 0 { dc.NumComplaints += len(comp) dc.NumComplainers += 1 } } gs.Counts = append(gs.Counts, dc) } if err := cdb.SaveGlobalStats(gs); err != nil { cdb.C.Errorf("Reset/Save fail, %v", err) } cdb.C.Infof("-- reset !--") //cdb.LoadGlobalStats(); }
func debugHandler(w http.ResponseWriter, r *http.Request) { s, e := date.WindowForYesterday() s = s.Add(-24 * time.Hour) e = e.Add(-24 * time.Hour) tStart := time.Now() procMap, err := GetProcedureMap(r, s, e) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } str := "OK!\n\n" str += fmt.Sprintf("* fetch+decode: %s\n* entries: %d\n\n", time.Since(tStart), len(procMap)) //for k,v := range procMap { str += fmt.Sprintf("%-10.10s %s\n", k, v) } w.Header().Set("Content-Type", "text/plain") w.Write([]byte(str)) }
// Examine all users. If they had any complaints, throw them in the queue. func bksvScanYesterdayHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) cdb := complaintdb.ComplaintDB{C: c, Memcache: true} var cps = []types.ComplainerProfile{} cps, err := cdb.GetAllProfiles() if err != nil { c.Errorf(" /bksv/scan-yesterday: getallprofiles: %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } start, end := date.WindowForYesterday() bksv_ok := 0 for _, cp := range cps { if cp.CcSfo == false { continue } var complaints = []types.Complaint{} complaints, err = cdb.GetComplaintsInSpanByEmailAddress(cp.EmailAddress, start, end) if err != nil { c.Errorf(" /bksv/scan-yesterday: getbyemail(%s): %v", cp.EmailAddress, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } if len(complaints) > 0 { t := taskqueue.NewPOSTTask("/bksv/submit-user", map[string][]string{ "user": {cp.EmailAddress}, }) if _, err := taskqueue.Add(c, t, "submitreports"); err != nil { c.Errorf(" /bksv/scan-yesterday: enqueue: %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } bksv_ok++ } } c.Infof("enqueued %d bksv", bksv_ok) w.Write([]byte(fmt.Sprintf("OK, enqueued %d", bksv_ok))) }
// This widget assumes the values 'date', 'range_from', and 'range_to' func FormValueDateRange(r *http.Request) (s, e time.Time, err error) { err = nil switch r.FormValue("date") { case "today": s, _ = date.WindowForToday() e = s case "yesterday": s, _ = date.WindowForYesterday() e = s case "range": s = date.ArbitraryDatestring2MidnightPdt(r.FormValue("range_from"), "2006/01/02") e = date.ArbitraryDatestring2MidnightPdt(r.FormValue("range_to"), "2006/01/02") if s.After(e) { s, e = e, s } } e = e.Add(23*time.Hour + 59*time.Minute + 59*time.Second) // make sure e covers its whole day return }
// We examine the tags CGI arg, which should be a pipe-delimited set of flight tags. func flightListHandler(w http.ResponseWriter, r *http.Request) { ctx := req2ctx(r) db := fdb.NewDB(r) tags := []string{} if r.FormValue("tags") != "" { tags = append(tags, strings.Split(r.FormValue("tags"), "|")...) } timeRange := regexp.MustCompile("/fdb/(.+)$").ReplaceAllString(r.URL.Path, "$1") var flights []ftype.Flight var err error switch timeRange { case "recent": flights, err = db.LookupRecentByTags(tags, 400) case "today": s, e := date.WindowForToday() flights, err = db.LookupTimeRangeByTags(tags, s, e) case "yesterday": s, e := date.WindowForYesterday() flights, err = db.LookupTimeRangeByTags(tags, s, e) } if err != nil { log.Errorf(ctx, " %s: %v", r.URL.Path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } var params = map[string]interface{}{ "Tags": tags, "TimeRange": timeRange, "Flights": flights2params(flights), } if err := templates.ExecuteTemplate(w, "fdb-recentlist", params); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } }
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) { k := fmt.Sprintf("%s:dailycounts", email) // The 'counts' is so we can have diff memcache objects c := []DailyCount{} cdb.Debugf("GDC_001", "GetDailyCounts() starting") // might return: datastore.ErrNoSuchEntity if dcs, err := cdb.fetchDailyCountSingleton(k); err == datastore.ErrNoSuchEntity { // Singleton not found; we don't care; treat same as empty singleton. } else if err != nil { cdb.Errorf("error getting item: %v", err) return c, err } else { c = dcs } cdb.Debugf("GDC_002", "singleton lookup done (%d entries)", len(c)) end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday start := end // by default, this will trigger no lookups (start=end means no missing) if len(c) > 0 { start = date.Datestring2MidnightPdt(c[0].Datestring) } else { cdb.Debugf("GDC_003", "counts empty ! track down oldest every, to start iteration range") if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil { cdb.Errorf("error looking up first complaint for %s: %v", email, err) return c, err } else if complaint != nil { // We move a day into the past; the algo below assumes we have data for the day 'start', // but in this case we don't; so trick it into generating data for today. start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1) } else { // cdb.Infof(" - lookup first ever, but empty\n") } cdb.Debugf("GDC_004", "start point found") } // Right after the first complaint: it set start to "now", but end is still yesterday. if start.After(end) { return c, nil } // We add a minute, to ensure that the day that contains 'end' is included missing := date.IntermediateMidnights(start, end.Add(time.Minute)) if len(missing) > 0 { for _, m := range missing { cdb.Debugf("GDC_005", "looking up a single span") dayStart, dayEnd := date.WindowForTime(m) if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil { return []DailyCount{}, err } else { c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false}) } } sort.Sort(DailyCountDesc(c)) // Now push back into datastore+memcache if err := cdb.putDailyCountSingleton(k, c); err != nil { cdb.Errorf("error storing counts singleton item: %v", err) } } cdb.Debugf("GDC_006", "all done") return c, nil }
func sendEmailsForYesterdayHandler(w http.ResponseWriter, r *http.Request) { start, end := date.WindowForYesterday() sendEmailsForWindow(w, r, start, end) }
func emailHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) session := sessions.Get(r) cdb := complaintdb.ComplaintDB{C: c} cp, err := cdb.GetProfileByEmailAddress(session.Values["email"].(string)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } start, end := date.WindowForYesterday() // end = time.Now() complaints, err2 := cdb.GetComplaintsInSpanByEmailAddress(cp.EmailAddress, start, end) if err2 != nil { http.Error(w, err2.Error(), http.StatusInternalServerError) return } var cap = types.ComplaintsAndProfile{ Profile: *cp, Complaints: complaints, } if err := templates.ExecuteTemplate(w, "email-update", map[string]interface{}{}); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } else { return } msg, err3 := GenerateEmail(c, cap) if err3 != nil { http.Error(w, err3.Error(), http.StatusInternalServerError) return } cap.Profile.CcSfo = true if len(cap.Complaints) == 0 { http.Error(w, "No complaints found ?!", http.StatusInternalServerError) return } msg2, err4 := GenerateSingleComplaintEmail(c, cap.Profile, cap.Complaints[len(cap.Complaints)-1]) if err4 != nil { http.Error(w, err4.Error(), http.StatusInternalServerError) return } var params = map[string]interface{}{ "Cap": cap, "EmailBundle": msg, "EmailSingle": msg2, "EmailBundleBody": template.HTML(msg.HTMLBody), "EmailSingleBody": template.HTML(msg2.HTMLBody), } /* if err4 := sendViaHTTPGateway(c, msg); err4 != nil { http.Error(w, err4.Error(), http.StatusInternalServerError) return } */ if err := templates.ExecuteTemplate(w, "email-debug", params); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } }
// http://stackoverflow.com/questions/13264555/store-an-object-in-memcache-of-gae-in-go func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) { // cdb.C.Infof("-- GetDaily for %s", email) k := fmt.Sprintf("%s:daily", email) c := []DailyCount{} if _, err := memcache.Gob.Get(cdb.C, k, &c); err == memcache.ErrCacheMiss { // cache miss, but we don't care } else if err != nil { cdb.C.Errorf("error getting item: %v", err) return c, err } end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday start := end // by default, this will trigger no lookups (start=end means no missing) if len(c) > 0 { start = date.Datestring2MidnightPdt(c[0].Datestring) } else { if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil { cdb.C.Errorf("error looking up first complaint for %s: %v", email, err) return c, err } else if complaint != nil { // We move a day into the past; the algo below assumes we have data for the day 'start', // but in this case we don't; so trick it into generating data for today. start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1) //cdb.C.Infof(" - lookup first ever, %s", complaint.Timestamp) } else { // cdb.C.Infof(" - lookup first ever, but empty\n") } } // Right after the first complaint: it set start to "now", but end is still yesterday. if start.After(end) { // cdb.C.Infof("--- s>e {%s} > {%s}\n", start, end) return c, nil } // We add a minute, to ensure that the day that contains 'end' is included missing := date.IntermediateMidnights(start, end.Add(time.Minute)) // cdb.C.Infof("--- missing? --- {%s} -> {%s} == %d\n", start, end.Add(time.Minute), len(missing)) if len(missing) > 0 { for _, m := range missing { dayStart, dayEnd := date.WindowForTime(m) if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil { return []DailyCount{}, err } else { // cdb.C.Infof(" - {%s} n=%d [%v]\n", dayStart, len(comp), m) c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false}) } } sort.Sort(DailyCountDesc(c)) // Now push back into memcache item := memcache.Item{Key: k, Object: c} if err := memcache.Gob.Set(cdb.C, &item); err != nil { cdb.C.Errorf("error setting item: %v", err) } } // cdb.C.Infof("--- done") return c, nil }