Exemple #1
0
func monthlySummaryTaskHandler(w http.ResponseWriter, r *http.Request) {
	month, year, err := FormValueMonthDefaultToPrev(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	countByUser := false
	now := date.NowInPdt()
	start := time.Date(int(year), time.Month(month), 1, 0, 0, 0, 0, now.Location())
	end := start.AddDate(0, 1, 0).Add(-1 * time.Second)

	bucketname := "serfr0-reports"
	filename := start.Format("summary-2006-01.txt")
	ctx := req2ctx(r)

	if exists, err := gcs.Exists(ctx, bucketname, filename); err != nil {
		http.Error(w, fmt.Sprintf("gcs.Exists=%v for gs://%s/%s (err=%v)", exists,
			bucketname, filename, err), http.StatusInternalServerError)
		return
	} else if exists {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte(fmt.Sprintf("OK!\nGCS file %s/%s already exists\n", bucketname, filename)))
		return
	}

	tStart := time.Now()
	str, err := SummaryReport(r, start, end, countByUser)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	gcsHandle, err := gcs.OpenRW(ctx, bucketname, filename, "text/plain")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	gcsHandle.IOWriter().Write([]byte(str))
	if err := gcsHandle.Close(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(fmt.Sprintf("OK!\nGCS monthly report %s/%s written, took %s",
		bucketname, filename, time.Since(tStart))))
}
Exemple #2
0
func generateMonthlyCSV(cdb complaintdb.ComplaintDB, month, year int) (string, int, error) {
	ctx := cdb.Ctx()
	bucketname := "serfr0-reports"

	now := date.NowInPdt()
	s := time.Date(int(year), time.Month(month), 1, 0, 0, 0, 0, now.Location())
	e := s.AddDate(0, 1, 0).Add(-1 * time.Second)
	log.Infof(ctx, "Starting /be/month: %s", s)

	// One time, at 00:00, for each day of the given month
	days := date.IntermediateMidnights(s.Add(-1*time.Second), e)

	filename := s.Format("complaints-20060102") + e.Format("-20060102.csv")

	gcsName := "gs://" + bucketname + "/" + filename

	if exists, err := gcs.Exists(ctx, bucketname, filename); err != nil {
		return gcsName, 0, fmt.Errorf("gcs.Exists=%v for gs://%s/%s (err=%v)", exists, bucketname, filename, err)
	} else if exists {
		return gcsName, 0, nil
	}

	gcsHandle, err := gcs.OpenRW(ctx, bucketname, filename, "text/plain")
	if err != nil {
		return gcsName, 0, err
	}
	csvWriter := csv.NewWriter(gcsHandle.IOWriter())

	cols := []string{
		"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
		"UnixEpoch", "Date", "Time(PDT)", "Notes", "ActivityDisturbed", "Flightnumber",
		"Notes",
		// Column names above are incorrect, but BKSV are used to them.
		//
		//"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
		//"UnixEpoch", "Date", "Time(PDT)", "Notes", "Flightnumber",
		//"ActivityDisturbed", "CcSFO",
	}
	csvWriter.Write(cols)

	tStart := time.Now()
	n := 0
	for _, dayStart := range days {
		dayEnd := dayStart.AddDate(0, 0, 1).Add(-1 * time.Second)
		log.Infof(ctx, " /be/month: %s - %s", dayStart, dayEnd)

		tIter := time.Now()
		iter := cdb.NewLongBatchingIter(cdb.QueryInSpan(dayStart, dayEnd))
		for {
			c, err := iter.NextWithErr()
			if err != nil {
				return gcsName, 0, fmt.Errorf("iterator failed after %s (%s): %v", err, time.Since(tIter),
					time.Since(tStart))
			}
			if c == nil {
				break
			}

			r := []string{
				c.Profile.CallerCode,
				c.Profile.FullName,
				c.Profile.Address,
				c.Profile.StructuredAddress.Zip,
				c.Profile.EmailAddress,
				fmt.Sprintf("%.4f", c.Profile.Lat),
				fmt.Sprintf("%.4f", c.Profile.Long),

				fmt.Sprintf("%d", c.Timestamp.UTC().Unix()),
				c.Timestamp.Format("2006/01/02"),
				c.Timestamp.Format("15:04:05"),
				c.Description,
				c.AircraftOverhead.FlightNumber,
				c.Activity,
				fmt.Sprintf("%v", c.Profile.CcSfo),
			}

			if err := csvWriter.Write(r); err != nil {
				return gcsName, 0, err
			}

			n++
		}
	}
	csvWriter.Flush()

	if err := gcsHandle.Close(); err != nil {
		return gcsName, 0, err
	}

	log.Infof(ctx, "monthly CSV successfully written to %s, %d rows", gcsName, n)

	return gcsName, n, nil
}
Exemple #3
0
func monthTaskHandler(w http.ResponseWriter, r *http.Request) {
	//ctx,_ := context.WithTimeout(appengine.NewContext(r), 599*time.Second)
	ctx := appengine.NewContext(r)

	cdb := complaintdb.ComplaintDB{
		//C: oldappengine.NewContext(r),
		C: oldappengine.Timeout(oldappengine.NewContext(r), 599*time.Second),
	}

	year, err := strconv.ParseInt(r.FormValue("year"), 10, 64)
	if err != nil {
		http.Error(w, "need arg 'year' (2015)", http.StatusInternalServerError)
		return
	}
	month, err := strconv.ParseInt(r.FormValue("month"), 10, 64)
	if err != nil {
		http.Error(w, "need arg 'month' (1-12)", http.StatusInternalServerError)
		return
	}

	now := date.NowInPdt()
	s := time.Date(int(year), time.Month(month), 1, 0, 0, 0, 0, now.Location())
	e := s.AddDate(0, 1, 0).Add(-1 * time.Second)
	log.Infof(ctx, "Starting /be/month: %s", s)

	// One time, at 00:00, for each day of the given month
	days := date.IntermediateMidnights(s.Add(-1*time.Second), e)

	filename := s.Format("complaints-20060102") + e.Format("-20060102.csv")

	gcsHandle, err := gcs.OpenRW(ctx, "serfr0-reports", filename, "text/plain")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	csvWriter := csv.NewWriter(gcsHandle.IOWriter())

	cols := []string{
		"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
		"UnixEpoch", "Date", "Time(PDT)", "Notes", "ActivityDisturbed", "Flightnumber",
		"Notes",
		// Column names above are incorrect, but BKSV are used to them.
		//
		//"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
		//"UnixEpoch", "Date", "Time(PDT)", "Notes", "Flightnumber",
		//"ActivityDisturbed", "CcSFO",
	}
	csvWriter.Write(cols)

	for _, dayStart := range days {
		dayEnd := dayStart.AddDate(0, 0, 1).Add(-1 * time.Second)
		log.Infof(ctx, " /be/month: %s - %s", dayStart, dayEnd)

		iter := cdb.NewIter(cdb.QueryInSpan(dayStart, dayEnd))
		for {
			c, err := iter.NextWithErr()
			if err != nil {
				http.Error(w, fmt.Sprintf("iterator failed: %v", err),
					http.StatusInternalServerError)
				return
			}
			if c == nil {
				break
			}

			r := []string{
				c.Profile.CallerCode,
				c.Profile.FullName,
				c.Profile.Address,
				c.Profile.StructuredAddress.Zip,
				c.Profile.EmailAddress,
				fmt.Sprintf("%.4f", c.Profile.Lat),
				fmt.Sprintf("%.4f", c.Profile.Long),

				fmt.Sprintf("%d", c.Timestamp.UTC().Unix()),
				c.Timestamp.Format("2006/01/02"),
				c.Timestamp.Format("15:04:05"),
				c.Description,
				c.AircraftOverhead.FlightNumber,
				c.Activity,
				fmt.Sprintf("%v", c.Profile.CcSfo),
			}

			if err := csvWriter.Write(r); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}
	}
	csvWriter.Flush()

	if err := gcsHandle.Close(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	log.Infof(ctx, "GCS report '%s' successfully written", filename)

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(fmt.Sprintf("OK!\nGCS file '%s' written to bucket", filename)))
}
Exemple #4
0
func reportWriter(c appengine.Context, w http.ResponseWriter, r *http.Request, s, e time.Time, opt ReportOptions, rep string, format string) {
	var rows []ReportRow
	var meta ReportMetadata
	var err error
	switch rep {
	case "classb":
		rows, meta, err = classbReport(c, s, e, opt)
	case "adsbclassb":
		rows, meta, err = adsbClassbReport(c, s, e, opt)
	case "discrep":
		rows, meta, err = discrepReport(c, s, e, opt)
	case "serfr1":
		rows, meta, err = serfr1Report(c, s, e, opt)
	case "brixx1":
		rows, meta, err = brixx1Report(c, s, e, opt)
	case "serfr1complaints":
		rows, meta, err = serfr1ComplaintsReport(c, s, e, opt)
	case "skimmer":
		rows, meta, err = skimmerReport(c, s, e, opt)
	case "brixxviolations":
		rows, meta, err = brixxViolationReport(c, s, e, opt)
	case "serfr1at":
		rows, meta, err = serfr1AtReport(c, s, e, opt)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Should do something better for this
	filename := date.NowInPdt().AddDate(0, 0, -1).Format(rep + "-20060102.csv")
	outFunc := func(csvWriter *csv.Writer) error {
		csvWriter.Write(rows[0].ToCSVHeaders())
		for _, r := range rows {
			if err := csvWriter.Write(r.ToCSV()); err != nil {
				return err
			}
		}
		csvWriter.Flush()
		return nil
	}

	if format == "csv" {
		w.Header().Set("Content-Type", "application/csv")
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
		csvWriter := csv.NewWriter(w)
		if err := outFunc(csvWriter); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

	} else if format == "gcs" {
		newCtx := newappengine.NewContext(r)
		handle, err := gcs.OpenRW(newCtx, "serfr0-reports", filename, "text/plain") //?"application/csv")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		csvWriter := csv.NewWriter(handle.IOWriter())
		if err := outFunc(csvWriter); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		if err := handle.Close(); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte(fmt.Sprintf("OK!\nGCS file '%s' written to bucket", filename)))

	} else {
		var params = map[string]interface{}{
			"Start":    s,
			"End":      e,
			"Metadata": meta,
			"Options":  opt,
		}

		// Is there not a more elegant way to do this kind of thing ?
		switch rep {
		case "classb":
			out := []CBRow{}
			for _, r := range rows {
				out = append(out, r.(CBRow))
			}
			params["Rows"] = out
		case "adsbclassb":
			out := []ACBRow{}
			for _, r := range rows {
				out = append(out, r.(ACBRow))
			}
			params["Rows"] = out
		case "serfr1", "brixx1", "discrep":
			out := []SERFR1Row{}
			for _, r := range rows {
				out = append(out, r.(SERFR1Row))
			}
			params["Rows"] = out
			rep = "serfr1"
		case "serfr1complaints":
			out := []SCRow{}
			for _, r := range rows {
				out = append(out, r.(SCRow))
			}
			params["Rows"] = out
		case "skimmer":
			out := []SkimRow{}
			for _, r := range rows {
				out = append(out, r.(SkimRow))
			}
			params["Rows"] = out
		case "brixxviolations":
			out := []BrixxRow{}
			for _, r := range rows {
				out = append(out, r.(BrixxRow))
			}
			params["Rows"] = out
		case "serfr1at":
			out := []SERFR1AtRow{}
			for _, r := range rows {
				out = append(out, r.(SERFR1AtRow))
			}
			params["Rows"] = out
		}

		if err := templates.ExecuteTemplate(w, "report-"+rep, params); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}
}
Exemple #5
0
// Returns number of records written (which is zero if the file already exists)
func writeAnonymizedGCSFile(r *http.Request, datestring, foldername, filename string) (int, error) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)

	// Get a list of users that as of right now, have opted out of data sharing.
	optOutUsers, err := cdb.GetComplainersCurrentlyOptedOut()
	if err != nil {
		return 0, fmt.Errorf("get optout users: %v", err)
	}

	if exists, err := gcs.Exists(ctx, foldername, filename); err != nil {
		return 0, err
	} else if exists {
		return 0, nil
	}

	gcsHandle, err := gcs.OpenRW(ctx, foldername, filename, "application/json")
	if err != nil {
		return 0, err
	}

	encoder := json.NewEncoder(gcsHandle.IOWriter())

	s := date.Datestring2MidnightPdt(datestring)
	e := s.AddDate(0, 0, 1).Add(-1 * time.Second) // +23:59:59 (or 22:59 or 24:59 when going in/out DST)

	n := 0
	// An iterator expires after 60s, no matter what; so carve up into short-lived iterators
	for _, dayWindow := range DayWindows(s, e) {
		iter := cdb.NewLongBatchingIter(cdb.QueryInSpan(dayWindow[0], dayWindow[1]))

		for {
			c, err := iter.NextWithErr()
			if err != nil {
				return 0, fmt.Errorf("iterator [%s,%s] failed at %s: %v",
					dayWindow[0], dayWindow[1], time.Now(), err)
			} else if c == nil {
				break // we're all done with this iterator
			}

			// If the user is currently opted out, ignore their data
			if _, exists := optOutUsers[c.Profile.EmailAddress]; exists {
				continue
			}

			n++
			ac := complaintdb.AnonymizeComplaint(c)

			if err := encoder.Encode(ac); err != nil {
				return 0, err
			}
		}
	}

	if err := gcsHandle.Close(); err != nil {
		return 0, err
	}

	log.Infof(ctx, "GCS bigquery file '%s' successfully written", filename)

	return n, nil
}