func SummaryReport(r *http.Request, start, end time.Time, countByUser bool) (string, error) {
ctx := req2ctx(r)
cdb := complaintdb.NewDB(ctx)
u := user.Current(cdb.Ctx())
str := ""
str += fmt.Sprintf("(t=%s)\n", time.Now())
str += fmt.Sprintf("Summary of disturbance reports:\n From [%s]\n To [%s]\n", start, end)
var countsByHour [24]int
countsByDate := map[string]int{}
countsByAirline := map[string]int{}
countsByEquip := map[string]int{}
countsByCity := map[string]int{}
countsByAirport := map[string]int{}
countsByProcedure := map[string]int{} // complaint counts, per arrival/departure procedure
flightCountsByProcedure := map[string]int{} // how many flights flew that procedure overall
proceduresByCity := map[string]map[string]int{} // For each city, breakdown by procedure
uniquesAll := map[string]int{}
uniquesPerDay := map[string]int{} // Each entry is a count for one unique user, for one day
uniquesByDate := map[string]map[string]int{}
uniquesByCity := map[string]map[string]int{}
uniquesPerDayByCity := map[string]map[string]int{} // [cityname][user:date] == daily_total
userHistsByCity := map[string]*histogram.Histogram{}
// An iterator expires after 60s, no matter what; so carve up into short-lived iterators
n := 0
for _, dayWindow := range DayWindows(start, end) {
// Get condensed flight data (for :NORCAL:)
flightsWithComplaintsButNoProcedureToday := map[string]int{}
cfMap, err := GetProcedureMap(r, dayWindow[0], dayWindow[1])
if err != nil {
return str, err
}
for _, cf := range cfMap {
if cf.Procedure.String() != "" {
flightCountsByProcedure[cf.Procedure.String()]++
}
}
iter := cdb.NewLongBatchingIter(cdb.QueryInSpan(dayWindow[0], dayWindow[1]))
cdb.Infof("running summary across %s-%s, for %v", dayWindow[0], dayWindow[1], u)
for {
c, err := iter.NextWithErr()
if err != nil {
return str, fmt.Errorf("iterator [%s,%s] failed at %s: %v",
dayWindow[0], dayWindow[1], time.Now(), err)
} else if c == nil {
break // we're all done with this iterator
}
n++
d := c.Timestamp.Format("2006.01.02")
uniquesAll[c.Profile.EmailAddress]++
uniquesPerDay[c.Profile.EmailAddress+":"+d]++
countsByHour[c.Timestamp.Hour()]++
countsByDate[d]++
if uniquesByDate[d] == nil {
uniquesByDate[d] = map[string]int{}
}
uniquesByDate[d][c.Profile.EmailAddress]++
if airline := c.AircraftOverhead.IATAAirlineCode(); airline != "" {
countsByAirline[airline]++
//dayCallsigns[c.AircraftOverhead.Callsign]++
if cf, exists := cfMap[c.AircraftOverhead.FlightNumber]; exists && cf.Procedure.String() != "" {
countsByProcedure[cf.Procedure.String()]++
} else {
countsByProcedure["procedure unknown"]++
flightsWithComplaintsButNoProcedureToday[c.AircraftOverhead.FlightNumber]++
}
whitelist := map[string]int{"SFO": 1, "SJC": 1, "OAK": 1}
if _, exists := whitelist[c.AircraftOverhead.Destination]; exists {
countsByAirport[fmt.Sprintf("%s arrival", c.AircraftOverhead.Destination)]++
} else if _, exists := whitelist[c.AircraftOverhead.Origin]; exists {
countsByAirport[fmt.Sprintf("%s departure", c.AircraftOverhead.Origin)]++
} else {
countsByAirport["airport unknown"]++ // overflights, and/or empty airport fields
}
} else {
countsByAirport["flight unidentified"]++
countsByProcedure["flight unidentified"]++
}
if city := c.Profile.GetStructuredAddress().City; city != "" {
countsByCity[city]++
if uniquesByCity[city] == nil {
uniquesByCity[city] = map[string]int{}
}
uniquesByCity[city][c.Profile.EmailAddress]++
if uniquesPerDayByCity[city] == nil {
uniquesPerDayByCity[city] = map[string]int{}
}
uniquesPerDayByCity[city][c.Profile.EmailAddress+":"+d]++
if proceduresByCity[city] == nil {
proceduresByCity[city] = map[string]int{}
}
if flightnumber := c.AircraftOverhead.FlightNumber; flightnumber != "" {
if cf, exists := cfMap[flightnumber]; exists && cf.Procedure.String() != "" {
proceduresByCity[city][cf.Procedure.Name]++
} else {
proceduresByCity[city]["proc?"]++
}
} else {
proceduresByCity[city]["flight?"]++
}
}
if equip := c.AircraftOverhead.EquipType; equip != "" {
countsByEquip[equip]++
}
}
unknowns := len(flightsWithComplaintsButNoProcedureToday)
flightCountsByProcedure["procedure unknown"] += unknowns
//for k,_ := range dayCallsigns { fmt.Fprintf(w, "** %s\n", k) }
}
// Generate histogram(s)
histByUser := histogram.Histogram{ValMax: 200, NumBuckets: 50}
for _, v := range uniquesPerDay {
histByUser.Add(histogram.ScalarVal(v))
}
for _, city := range keysByIntValDesc(countsByCity) {
if userHistsByCity[city] == nil {
userHistsByCity[city] = &histogram.Histogram{ValMax: 200, NumBuckets: 50}
}
for _, n := range uniquesPerDayByCity[city] {
userHistsByCity[city].Add(histogram.ScalarVal(n))
}
}
str += fmt.Sprintf("\nTotals:\n Days : %d\n"+
" Disturbance reports : %d\n People reporting : %d\n",
len(countsByDate), n, len(uniquesAll))
str += fmt.Sprintf("\nComplaints per user, histogram (0-200):\n %s\n", histByUser)
/*
str += fmt.Sprintf("\n[BETA: no more than 80%% accurate!] Disturbance reports, "+
"counted by procedure type, breaking out vectored flights "+
"(e.g. PROCEDURE/LAST-ON-PROCEDURE-WAYPOINT):\n")
for _,k := range keysByKeyAsc(countsByProcedure) {
avg := 0.0
if flightCountsByProcedure[k] > 0 {
avg = float64(countsByProcedure[k]) / float64(flightCountsByProcedure[k])
}
str += fmt.Sprintf(" %-20.20s: %6d (%5d such flights with complaints; %3.0f complaints/flight)\n",
k, countsByProcedure[k], flightCountsByProcedure[k], avg)
}
*/
str += fmt.Sprintf("\nDisturbance reports, counted by airport:\n")
for _, k := range keysByKeyAsc(countsByAirport) {
str += fmt.Sprintf(" %-20.20s: %6d\n", k, countsByAirport[k])
}
str += fmt.Sprintf("\nDisturbance reports, counted by City (where known):\n")
for _, k := range keysByIntValDesc(countsByCity) {
str += fmt.Sprintf(" %-40.40s: %5d (%4d people reporting)\n",
k, countsByCity[k], len(uniquesByCity[k]))
}
str += fmt.Sprintf("\nDisturbance reports, as per-user-per-day histograms, by City (where known):\n")
for _, k := range keysByIntValDesc(countsByCity) {
str += fmt.Sprintf(" %-40.40s: %s\n", k, userHistsByCity[k])
}
/*
str += fmt.Sprintf("\nDisturbance reports, counted by City & procedure type (where known):\n")
for _,k := range keysByIntValDesc(countsByCity) {
pStr := fmt.Sprintf("SERFR: %.0f%%, non-SERFR: %.0f%%, flight unknown: %.0f%%",
100.0 * (float64(proceduresByCity[k]["SERFR2"]) / float64(countsByCity[k])),
100.0 * (float64(proceduresByCity[k]["proc?"]) / float64(countsByCity[k])),
100.0 * (float64(proceduresByCity[k]["flight?"]) / float64(countsByCity[k])))
str += fmt.Sprintf(" %-40.40s: %5d (%4d people reporting) (%s)\n",
k, countsByCity[k], len(uniquesByCity[k]), pStr)
}
*/
str += fmt.Sprintf("\nDisturbance reports, counted by date:\n")
for _, k := range keysByKeyAsc(countsByDate) {
str += fmt.Sprintf(" %s: %5d (%4d people reporting)\n", k, countsByDate[k], len(uniquesByDate[k]))
}
str += fmt.Sprintf("\nDisturbance reports, counted by aircraft equipment type (where known):\n")
for _, k := range keysByIntValDesc(countsByEquip) {
if countsByEquip[k] < 5 {
break
}
str += fmt.Sprintf(" %-40.40s: %5d\n", k, countsByEquip[k])
}
str += fmt.Sprintf("\nDisturbance reports, counted by Airline (where known):\n")
for _, k := range keysByIntValDesc(countsByAirline) {
if countsByAirline[k] < 5 || len(k) > 2 {
continue
}
str += fmt.Sprintf(" %s: %6d\n", k, countsByAirline[k])
}
str += fmt.Sprintf("\nDisturbance reports, counted by hour of day (across all dates):\n")
for i, n := range countsByHour {
str += fmt.Sprintf(" %02d: %5d\n", i, n)
}
if countByUser {
str += fmt.Sprintf("\nDisturbance reports, counted by user:\n")
for _, k := range keysByIntValDesc(uniquesAll) {
str += fmt.Sprintf(" %-60.60s: %5d\n", k, uniquesAll[k])
}
}
str += fmt.Sprintf("(t=%s)\n", time.Now())
return str, nil
}
func classbReport(c appengine.Context, s, e time.Time, opt ReportOptions) ([]ReportRow, ReportMetadata, error) {
fdb := fdb.FlightDB{C: c}
maybeMemcache(&fdb, e)
tags := []string{
flightdb.KTagReliableClassBViolation,
flightdb.KTagLocalADSBClassBViolation,
}
meta := ReportMetadata{}
h := histogram.Histogram{} // Only use it for the stats
rows := []ReportRow{}
seen := map[string]bool{} // Because we do two passes, we might see same flight twice.
metars, err := metar.FetchFromNOAA(urlfetch.Client(c), "KSFO",
s.Add(-6*time.Hour), e.Add(6*time.Hour))
if err != nil {
return rows, meta, err
}
reportFunc := func(f *flightdb.Flight) {
if _, exists := seen[f.UniqueIdentifier()]; exists {
return
} else {
seen[f.UniqueIdentifier()] = true
}
bestTrack := "FA"
if f.HasTag(flightdb.KTagLocalADSBClassBViolation) {
bestTrack = "ADSB"
}
_, cbt := f.SFOClassB(bestTrack, metars)
tmpRows := []ReportRow{}
seq := 0
for _, cbtp := range cbt {
if cbtp.A.IsViolation() {
fClone := f.ShallowCopy()
tmpRows = append(tmpRows, CBRow{seq, flight2Url(*fClone), *fClone, cbtp.TP, cbtp.A})
seq++
}
}
if len(tmpRows) == 0 {
return
}
worstCBRow := tmpRows[0].(CBRow)
if seq > 0 {
// Select the worst row
n, belowBy := 0, 0.0
for i, row := range tmpRows {
if row.(CBRow).A.BelowBy > belowBy {
n, belowBy = i, row.(CBRow).A.BelowBy
}
}
worstCBRow = tmpRows[n].(CBRow)
worstCBRow.Seq = 0 // fake this out for the webpage
}
if opt.ClassB_LocalDataOnly && !f.HasTag(flightdb.KTagLocalADSBClassBViolation) {
meta["[C] -- Skippped; not local - "+worstCBRow.TP.LongSource()]++
} else {
meta["[C] -- Detected via "+worstCBRow.TP.LongSource()]++
h.Add(histogram.ScalarVal(worstCBRow.A.BelowBy))
if opt.ClassB_OnePerFlight {
rows = append(rows, worstCBRow)
} else {
rows = append(rows, tmpRows...)
}
}
}
// Need to do multiple passes, because of tagA-or-tagB sillness
// In each case, limit to SERFR1 flights
for _, tag := range tags {
theseTags := []string{tag, flightdb.KTagSERFR1}
if err := fdb.IterWith(fdb.QueryTimeRangeByTags(theseTags, s, e), reportFunc); err != nil {
return nil, nil, err
}
}
if n, err := fdb.CountTimeRangeByTags([]string{flightdb.KTagSERFR1}, s, e); err != nil {
return nil, nil, err
} else {
meta["[A] Total SERFR1 Flights"] = float64(n)
}
if stats, valid := h.Stats(); valid {
meta["[B] Num violating flights"] = float64(stats.N)
meta["[D] Mean violation below Class B floor"] = float64(int(stats.Mean))
meta["[D] Stddev"] = float64(int(stats.Stddev))
} else {
meta["[B] Num violating flights"] = 0.0
}
return rows, meta, nil
}