Example #1
0
func cannedSerfr1ComplaintsHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	reportWriter(c, w, r, s, e, ReportOptions{}, "serfr1complaints", format)
}
Example #2
0
func cannedDiscrepHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	opt := ReportOptions{}

	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	reportWriter(appengine.NewContext(r), w, r, s, e, opt, "discrep", format)
}
Example #3
0
func cannedAdsbClassBHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	opt := ReportOptions{}
	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	reportWriter(c, w, r, s, e, opt, "adsbclassb", format)
}
Example #4
0
func debugHandler3(w http.ResponseWriter, r *http.Request) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)
	tStart := time.Now()

	start, end := date.WindowForYesterday()
	keys, err := cdb.GetComplaintKeysInSpan(start, end)

	str := fmt.Sprintf("OK\nret: %d\nerr: %v\nElapsed: %s\ns: %s\ne: %s\n",
		len(keys), err, time.Since(tStart), start, end)

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(str))
}
Example #5
0
func cannedSerfr1Handler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	opt := ReportOptions{
		ClassB_OnePerFlight: true,
	}

	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	reportWriter(appengine.NewContext(r), w, r, s, e, opt, "serfr1", format)
}
Example #6
0
func (cdb ComplaintDB) ResetGlobalStats() {
	if err := cdb.DeletAllGlobalStats(); err != nil {
		cdb.Errorf("Reset/DeleteAll fail, %v", err)
		return
	}

	profiles, err := cdb.GetAllProfiles()
	if err != nil {
		return
	}

	// Upon reset (writing a fresh new singleton), we need to generate a key
	rootKey := datastore.NewKey(cdb.Ctx(), kGlobalStatsKind, "foo", 0, nil)
	key := datastore.NewIncompleteKey(cdb.Ctx(), kGlobalStatsKind, rootKey)
	gs := GlobalStats{
		DatastoreKey: key.Encode(),
	}

	// This is too slow to recalculate this way; it runs into the 10m timeout
	//start := date.ArbitraryDatestring2MidnightPdt("2015/08/09", "2006/01/02").Add(-1 * time.Second)
	start := date.ArbitraryDatestring2MidnightPdt("2016/03/15", "2006/01/02").Add(-1 * time.Second)
	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday

	midnights := date.IntermediateMidnights(start, end.Add(time.Minute))
	for _, m := range midnights {
		dayStart, dayEnd := date.WindowForTime(m)

		dc := DailyCount{Datestring: date.Time2Datestring(dayStart)}

		for _, p := range profiles {
			if keys, err := cdb.GetComplaintKeysInSpanByEmailAddress(dayStart, dayEnd, p.EmailAddress); err != nil {
				cdb.Errorf("Reset/Lookup fail, %v", err)
			} else if len(keys) > 0 {
				dc.NumComplaints += len(keys)
				dc.NumComplainers += 1
			}
		}
		gs.Counts = append(gs.Counts, dc)
	}

	if err := cdb.SaveGlobalStats(gs); err != nil {
		cdb.Errorf("Reset/Save fail, %v", err)
	}
}
Example #7
0
func bksvSubmitUserHandler(w http.ResponseWriter, r *http.Request) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)
	client := cdb.HTTPClient()
	start, end := date.WindowForYesterday()
	bksv_ok, bksv_not_ok := 0, 0

	email := r.FormValue("user")

	if cp, err := cdb.GetProfileByEmailAddress(email); err != nil {
		cdb.Errorf(" /bksv/submit-user(%s): getprofile: %v", email, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return

	} else if complaints, err := cdb.GetComplaintsInSpanByEmailAddress(email, start, end); err != nil {
		cdb.Errorf(" /bksv/submit-user(%s): getcomplaints: %v", email, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return

	} else {
		for i, complaint := range complaints {
			time.Sleep(time.Millisecond * 200)
			if debug, err := bksv.PostComplaint(client, *cp, complaint); err != nil {
				//cdb.Infof("pro: %v", cp)
				//cdb.Infof("comp: %#v", complaint)
				cdb.Errorf("BKSV posting error: %v", err)
				cdb.Infof("BKSV Debug\n------\n%s\n------\n", debug)
				bksv_not_ok++
			} else {
				if i == 0 {
					cdb.Infof("BKSV [OK] Debug\n------\n%s\n------\n", debug)
				}
				bksv_ok++
			}
		}
	}

	cdb.Infof("bksv for %s, %d/%d", email, bksv_ok, bksv_not_ok)
	if bksv_not_ok > 0 {
		cdb.Errorf("bksv for %s, %d/%d", email, bksv_ok, bksv_not_ok)
	}
	w.Write([]byte("OK"))
}
Example #8
0
func (cdb ComplaintDB) ResetGlobalStats() {
	if err := cdb.DeletAllGlobalStats(); err != nil {
		cdb.C.Errorf("Reset/DeleteAll fail, %v", err)
		return
	}

	profiles, err := cdb.GetAllProfiles()
	if err != nil {
		return
	}

	// Upon reset (writing a fresh new singleton), we need to generate a key
	rootKey := datastore.NewKey(cdb.C, kGlobalStatsKind, "foo", 0, nil)
	key := datastore.NewIncompleteKey(cdb.C, kGlobalStatsKind, rootKey)
	gs := GlobalStats{
		DatastoreKey: key.Encode(),
	}

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end.AddDate(0, 0, -100)
	midnights := date.IntermediateMidnights(start, end.Add(time.Minute))
	for _, m := range midnights {
		dayStart, dayEnd := date.WindowForTime(m)

		dc := DailyCount{Datestring: date.Time2Datestring(dayStart)}

		for _, p := range profiles {
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(p.EmailAddress, dayStart, dayEnd); err != nil {
				cdb.C.Errorf("Reset/Lookup fail, %v", err)
			} else if len(comp) > 0 {
				dc.NumComplaints += len(comp)
				dc.NumComplainers += 1
			}
		}
		gs.Counts = append(gs.Counts, dc)
	}

	if err := cdb.SaveGlobalStats(gs); err != nil {
		cdb.C.Errorf("Reset/Save fail, %v", err)
	}
	cdb.C.Infof("-- reset !--")
	//cdb.LoadGlobalStats();
}
Example #9
0
func debugHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	s = s.Add(-24 * time.Hour)
	e = e.Add(-24 * time.Hour)

	tStart := time.Now()
	procMap, err := GetProcedureMap(r, s, e)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	str := "OK!\n\n"
	str += fmt.Sprintf("* fetch+decode: %s\n* entries: %d\n\n", time.Since(tStart), len(procMap))

	//for k,v := range procMap { str += fmt.Sprintf("%-10.10s %s\n", k, v) }

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(str))
}
Example #10
0
// Examine all users. If they had any complaints, throw them in the queue.
func bksvScanYesterdayHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	cdb := complaintdb.ComplaintDB{C: c, Memcache: true}
	var cps = []types.ComplainerProfile{}
	cps, err := cdb.GetAllProfiles()
	if err != nil {
		c.Errorf(" /bksv/scan-yesterday: getallprofiles: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	start, end := date.WindowForYesterday()
	bksv_ok := 0

	for _, cp := range cps {
		if cp.CcSfo == false {
			continue
		}

		var complaints = []types.Complaint{}
		complaints, err = cdb.GetComplaintsInSpanByEmailAddress(cp.EmailAddress, start, end)
		if err != nil {
			c.Errorf(" /bksv/scan-yesterday: getbyemail(%s): %v", cp.EmailAddress, err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if len(complaints) > 0 {
			t := taskqueue.NewPOSTTask("/bksv/submit-user", map[string][]string{
				"user": {cp.EmailAddress},
			})
			if _, err := taskqueue.Add(c, t, "submitreports"); err != nil {
				c.Errorf(" /bksv/scan-yesterday: enqueue: %v", err)
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			bksv_ok++
		}
	}
	c.Infof("enqueued %d bksv", bksv_ok)
	w.Write([]byte(fmt.Sprintf("OK, enqueued %d", bksv_ok)))
}
Example #11
0
// This widget assumes the values 'date', 'range_from', and 'range_to'
func FormValueDateRange(r *http.Request) (s, e time.Time, err error) {
	err = nil

	switch r.FormValue("date") {
	case "today":
		s, _ = date.WindowForToday()
		e = s
	case "yesterday":
		s, _ = date.WindowForYesterday()
		e = s
	case "range":
		s = date.ArbitraryDatestring2MidnightPdt(r.FormValue("range_from"), "2006/01/02")
		e = date.ArbitraryDatestring2MidnightPdt(r.FormValue("range_to"), "2006/01/02")
		if s.After(e) {
			s, e = e, s
		}
	}

	e = e.Add(23*time.Hour + 59*time.Minute + 59*time.Second) // make sure e covers its whole day

	return
}
Example #12
0
// We examine the tags CGI arg, which should be a pipe-delimited set of flight tags.
func flightListHandler(w http.ResponseWriter, r *http.Request) {
	ctx := req2ctx(r)
	db := fdb.NewDB(r)

	tags := []string{}
	if r.FormValue("tags") != "" {
		tags = append(tags, strings.Split(r.FormValue("tags"), "|")...)
	}

	timeRange := regexp.MustCompile("/fdb/(.+)$").ReplaceAllString(r.URL.Path, "$1")
	var flights []ftype.Flight
	var err error

	switch timeRange {
	case "recent":
		flights, err = db.LookupRecentByTags(tags, 400)
	case "today":
		s, e := date.WindowForToday()
		flights, err = db.LookupTimeRangeByTags(tags, s, e)
	case "yesterday":
		s, e := date.WindowForYesterday()
		flights, err = db.LookupTimeRangeByTags(tags, s, e)
	}

	if err != nil {
		log.Errorf(ctx, " %s: %v", r.URL.Path, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var params = map[string]interface{}{
		"Tags":      tags,
		"TimeRange": timeRange,
		"Flights":   flights2params(flights),
	}
	if err := templates.ExecuteTemplate(w, "fdb-recentlist", params); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
Example #13
0
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) {
	k := fmt.Sprintf("%s:dailycounts", email) // The 'counts' is so we can have diff memcache objects
	c := []DailyCount{}

	cdb.Debugf("GDC_001", "GetDailyCounts() starting")

	// 	might return: datastore.ErrNoSuchEntity
	if dcs, err := cdb.fetchDailyCountSingleton(k); err == datastore.ErrNoSuchEntity {
		// Singleton not found; we don't care; treat same as empty singleton.
	} else if err != nil {
		cdb.Errorf("error getting item: %v", err)
		return c, err
	} else {
		c = dcs
	}
	cdb.Debugf("GDC_002", "singleton lookup done (%d entries)", len(c))

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end                        // by default, this will trigger no lookups (start=end means no missing)

	if len(c) > 0 {
		start = date.Datestring2MidnightPdt(c[0].Datestring)
	} else {
		cdb.Debugf("GDC_003", "counts empty ! track down oldest every, to start iteration range")
		if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil {
			cdb.Errorf("error looking up first complaint for %s: %v", email, err)
			return c, err
		} else if complaint != nil {
			// We move a day into the past; the algo below assumes we have data for the day 'start',
			// but in this case we don't; so trick it into generating data for today.
			start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1)
		} else {
			// cdb.Infof("  - lookup first ever, but empty\n")
		}
		cdb.Debugf("GDC_004", "start point found")
	}

	// Right after the first complaint: it set start to "now", but end is still yesterday.
	if start.After(end) {
		return c, nil
	}

	// We add a minute, to ensure that the day that contains 'end' is included
	missing := date.IntermediateMidnights(start, end.Add(time.Minute))
	if len(missing) > 0 {
		for _, m := range missing {
			cdb.Debugf("GDC_005", "looking up a single span")
			dayStart, dayEnd := date.WindowForTime(m)
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil {
				return []DailyCount{}, err
			} else {
				c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false})
			}
		}
		sort.Sort(DailyCountDesc(c))

		// Now push back into datastore+memcache
		if err := cdb.putDailyCountSingleton(k, c); err != nil {
			cdb.Errorf("error storing counts singleton item: %v", err)
		}
	}
	cdb.Debugf("GDC_006", "all done")

	return c, nil
}
Example #14
0
func sendEmailsForYesterdayHandler(w http.ResponseWriter, r *http.Request) {
	start, end := date.WindowForYesterday()
	sendEmailsForWindow(w, r, start, end)
}
Example #15
0
func emailHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	session := sessions.Get(r)
	cdb := complaintdb.ComplaintDB{C: c}

	cp, err := cdb.GetProfileByEmailAddress(session.Values["email"].(string))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	start, end := date.WindowForYesterday()
	// end = time.Now()
	complaints, err2 := cdb.GetComplaintsInSpanByEmailAddress(cp.EmailAddress, start, end)
	if err2 != nil {
		http.Error(w, err2.Error(), http.StatusInternalServerError)
		return
	}

	var cap = types.ComplaintsAndProfile{
		Profile:    *cp,
		Complaints: complaints,
	}

	if err := templates.ExecuteTemplate(w, "email-update", map[string]interface{}{}); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else {
		return
	}

	msg, err3 := GenerateEmail(c, cap)
	if err3 != nil {
		http.Error(w, err3.Error(), http.StatusInternalServerError)
		return
	}
	cap.Profile.CcSfo = true
	if len(cap.Complaints) == 0 {
		http.Error(w, "No complaints found ?!", http.StatusInternalServerError)
		return
	}
	msg2, err4 := GenerateSingleComplaintEmail(c, cap.Profile, cap.Complaints[len(cap.Complaints)-1])
	if err4 != nil {
		http.Error(w, err4.Error(), http.StatusInternalServerError)
		return
	}

	var params = map[string]interface{}{
		"Cap":             cap,
		"EmailBundle":     msg,
		"EmailSingle":     msg2,
		"EmailBundleBody": template.HTML(msg.HTMLBody),
		"EmailSingleBody": template.HTML(msg2.HTMLBody),
	}

	/* if err4 := sendViaHTTPGateway(c, msg); err4 != nil {
		http.Error(w, err4.Error(), http.StatusInternalServerError)
		return
	} */

	if err := templates.ExecuteTemplate(w, "email-debug", params); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
Example #16
0
// http://stackoverflow.com/questions/13264555/store-an-object-in-memcache-of-gae-in-go
func (cdb *ComplaintDB) GetDailyCounts(email string) ([]DailyCount, error) {
	// cdb.C.Infof("-- GetDaily for %s", email)

	k := fmt.Sprintf("%s:daily", email)
	c := []DailyCount{}

	if _, err := memcache.Gob.Get(cdb.C, k, &c); err == memcache.ErrCacheMiss {
		// cache miss, but we don't care
	} else if err != nil {
		cdb.C.Errorf("error getting item: %v", err)
		return c, err
	}

	end, _ := date.WindowForYesterday() // end is the final day we count for; yesterday
	start := end                        // by default, this will trigger no lookups (start=end means no missing)

	if len(c) > 0 {
		start = date.Datestring2MidnightPdt(c[0].Datestring)
	} else {
		if complaint, err := cdb.GetOldestComplaintByEmailAddress(email); err != nil {
			cdb.C.Errorf("error looking up first complaint for %s: %v", email, err)
			return c, err
		} else if complaint != nil {
			// We move a day into the past; the algo below assumes we have data for the day 'start',
			// but in this case we don't; so trick it into generating data for today.
			start = date.AtLocalMidnight(complaint.Timestamp).AddDate(0, 0, -1)
			//cdb.C.Infof("  - lookup first ever, %s", complaint.Timestamp)
		} else {
			// cdb.C.Infof("  - lookup first ever, but empty\n")
		}
	}

	// Right after the first complaint: it set start to "now", but end is still yesterday.
	if start.After(end) {
		// cdb.C.Infof("--- s>e {%s} > {%s}\n", start, end)
		return c, nil
	}

	// We add a minute, to ensure that the day that contains 'end' is included
	missing := date.IntermediateMidnights(start, end.Add(time.Minute))
	// cdb.C.Infof("--- missing? --- {%s} -> {%s} == %d\n", start, end.Add(time.Minute), len(missing))
	if len(missing) > 0 {
		for _, m := range missing {
			dayStart, dayEnd := date.WindowForTime(m)
			if comp, err := cdb.GetComplaintsInSpanByEmailAddress(email, dayStart, dayEnd); err != nil {
				return []DailyCount{}, err
			} else {
				// cdb.C.Infof("  -  {%s}  n=%d [%v]\n", dayStart, len(comp), m)
				c = append(c, DailyCount{date.Time2Datestring(dayStart), len(comp), 1, false, false})
			}
		}
		sort.Sort(DailyCountDesc(c))

		// Now push back into memcache
		item := memcache.Item{Key: k, Object: c}
		if err := memcache.Gob.Set(cdb.C, &item); err != nil {
			cdb.C.Errorf("error setting item: %v", err)
		}
	}

	// cdb.C.Infof("--- done")
	return c, nil
}