Пример #1
0
func SummaryReport(r *http.Request, start, end time.Time, countByUser bool) (string, error) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)
	u := user.Current(cdb.Ctx())

	str := ""
	str += fmt.Sprintf("(t=%s)\n", time.Now())
	str += fmt.Sprintf("Summary of disturbance reports:\n From [%s]\n To   [%s]\n", start, end)

	var countsByHour [24]int
	countsByDate := map[string]int{}
	countsByAirline := map[string]int{}
	countsByEquip := map[string]int{}
	countsByCity := map[string]int{}
	countsByAirport := map[string]int{}

	countsByProcedure := map[string]int{}           // complaint counts, per arrival/departure procedure
	flightCountsByProcedure := map[string]int{}     // how many flights flew that procedure overall
	proceduresByCity := map[string]map[string]int{} // For each city, breakdown by procedure

	uniquesAll := map[string]int{}
	uniquesPerDay := map[string]int{} // Each entry is a count for one unique user, for one day
	uniquesByDate := map[string]map[string]int{}
	uniquesByCity := map[string]map[string]int{}

	uniquesPerDayByCity := map[string]map[string]int{} // [cityname][user:date] == daily_total

	userHistsByCity := map[string]*histogram.Histogram{}

	// An iterator expires after 60s, no matter what; so carve up into short-lived iterators
	n := 0
	for _, dayWindow := range DayWindows(start, end) {

		// Get condensed flight data (for :NORCAL:)
		flightsWithComplaintsButNoProcedureToday := map[string]int{}
		cfMap, err := GetProcedureMap(r, dayWindow[0], dayWindow[1])
		if err != nil {
			return str, err
		}
		for _, cf := range cfMap {
			if cf.Procedure.String() != "" {
				flightCountsByProcedure[cf.Procedure.String()]++
			}
		}

		iter := cdb.NewLongBatchingIter(cdb.QueryInSpan(dayWindow[0], dayWindow[1]))
		cdb.Infof("running summary across %s-%s, for %v", dayWindow[0], dayWindow[1], u)

		for {
			c, err := iter.NextWithErr()
			if err != nil {
				return str, fmt.Errorf("iterator [%s,%s] failed at %s: %v",
					dayWindow[0], dayWindow[1], time.Now(), err)
			} else if c == nil {
				break // we're all done with this iterator
			}

			n++
			d := c.Timestamp.Format("2006.01.02")

			uniquesAll[c.Profile.EmailAddress]++
			uniquesPerDay[c.Profile.EmailAddress+":"+d]++
			countsByHour[c.Timestamp.Hour()]++
			countsByDate[d]++
			if uniquesByDate[d] == nil {
				uniquesByDate[d] = map[string]int{}
			}
			uniquesByDate[d][c.Profile.EmailAddress]++

			if airline := c.AircraftOverhead.IATAAirlineCode(); airline != "" {
				countsByAirline[airline]++
				//dayCallsigns[c.AircraftOverhead.Callsign]++

				if cf, exists := cfMap[c.AircraftOverhead.FlightNumber]; exists && cf.Procedure.String() != "" {
					countsByProcedure[cf.Procedure.String()]++
				} else {
					countsByProcedure["procedure unknown"]++
					flightsWithComplaintsButNoProcedureToday[c.AircraftOverhead.FlightNumber]++
				}

				whitelist := map[string]int{"SFO": 1, "SJC": 1, "OAK": 1}
				if _, exists := whitelist[c.AircraftOverhead.Destination]; exists {
					countsByAirport[fmt.Sprintf("%s arrival", c.AircraftOverhead.Destination)]++
				} else if _, exists := whitelist[c.AircraftOverhead.Origin]; exists {
					countsByAirport[fmt.Sprintf("%s departure", c.AircraftOverhead.Origin)]++
				} else {
					countsByAirport["airport unknown"]++ // overflights, and/or empty airport fields
				}
			} else {
				countsByAirport["flight unidentified"]++
				countsByProcedure["flight unidentified"]++
			}

			if city := c.Profile.GetStructuredAddress().City; city != "" {
				countsByCity[city]++

				if uniquesByCity[city] == nil {
					uniquesByCity[city] = map[string]int{}
				}
				uniquesByCity[city][c.Profile.EmailAddress]++

				if uniquesPerDayByCity[city] == nil {
					uniquesPerDayByCity[city] = map[string]int{}
				}
				uniquesPerDayByCity[city][c.Profile.EmailAddress+":"+d]++

				if proceduresByCity[city] == nil {
					proceduresByCity[city] = map[string]int{}
				}
				if flightnumber := c.AircraftOverhead.FlightNumber; flightnumber != "" {
					if cf, exists := cfMap[flightnumber]; exists && cf.Procedure.String() != "" {
						proceduresByCity[city][cf.Procedure.Name]++
					} else {
						proceduresByCity[city]["proc?"]++
					}
				} else {
					proceduresByCity[city]["flight?"]++
				}
			}
			if equip := c.AircraftOverhead.EquipType; equip != "" {
				countsByEquip[equip]++
			}
		}

		unknowns := len(flightsWithComplaintsButNoProcedureToday)
		flightCountsByProcedure["procedure unknown"] += unknowns

		//for k,_ := range dayCallsigns { fmt.Fprintf(w, "** %s\n", k) }
	}

	// Generate histogram(s)
	histByUser := histogram.Histogram{ValMax: 200, NumBuckets: 50}
	for _, v := range uniquesPerDay {
		histByUser.Add(histogram.ScalarVal(v))
	}

	for _, city := range keysByIntValDesc(countsByCity) {
		if userHistsByCity[city] == nil {
			userHistsByCity[city] = &histogram.Histogram{ValMax: 200, NumBuckets: 50}
		}
		for _, n := range uniquesPerDayByCity[city] {
			userHistsByCity[city].Add(histogram.ScalarVal(n))
		}
	}

	str += fmt.Sprintf("\nTotals:\n Days                : %d\n"+
		" Disturbance reports : %d\n People reporting    : %d\n",
		len(countsByDate), n, len(uniquesAll))

	str += fmt.Sprintf("\nComplaints per user, histogram (0-200):\n %s\n", histByUser)
	/*
		str += fmt.Sprintf("\n[BETA: no more than 80%% accurate!] Disturbance reports, "+
			"counted by procedure type, breaking out vectored flights "+
			"(e.g. PROCEDURE/LAST-ON-PROCEDURE-WAYPOINT):\n")
		for _,k := range keysByKeyAsc(countsByProcedure) {
			avg := 0.0
			if flightCountsByProcedure[k] > 0 {
				avg = float64(countsByProcedure[k]) / float64(flightCountsByProcedure[k])
			}
			str += fmt.Sprintf(" %-20.20s: %6d (%5d such flights with complaints; %3.0f complaints/flight)\n",
				k, countsByProcedure[k], flightCountsByProcedure[k], avg)
		}
	*/

	str += fmt.Sprintf("\nDisturbance reports, counted by airport:\n")
	for _, k := range keysByKeyAsc(countsByAirport) {
		str += fmt.Sprintf(" %-20.20s: %6d\n", k, countsByAirport[k])
	}

	str += fmt.Sprintf("\nDisturbance reports, counted by City (where known):\n")
	for _, k := range keysByIntValDesc(countsByCity) {
		str += fmt.Sprintf(" %-40.40s: %5d (%4d people reporting)\n",
			k, countsByCity[k], len(uniquesByCity[k]))
	}

	str += fmt.Sprintf("\nDisturbance reports, as per-user-per-day histograms, by City (where known):\n")
	for _, k := range keysByIntValDesc(countsByCity) {
		str += fmt.Sprintf(" %-40.40s: %s\n", k, userHistsByCity[k])
	}

	/*
		str += fmt.Sprintf("\nDisturbance reports, counted by City & procedure type (where known):\n")
		for _,k := range keysByIntValDesc(countsByCity) {
			pStr := fmt.Sprintf("SERFR: %.0f%%, non-SERFR: %.0f%%, flight unknown: %.0f%%",
				100.0 * (float64(proceduresByCity[k]["SERFR2"]) / float64(countsByCity[k])),
				100.0 * (float64(proceduresByCity[k]["proc?"]) / float64(countsByCity[k])),
				100.0 * (float64(proceduresByCity[k]["flight?"]) / float64(countsByCity[k])))
			str += fmt.Sprintf(" %-40.40s: %5d (%4d people reporting) (%s)\n",
				k, countsByCity[k], len(uniquesByCity[k]), pStr)
		}
	*/

	str += fmt.Sprintf("\nDisturbance reports, counted by date:\n")
	for _, k := range keysByKeyAsc(countsByDate) {
		str += fmt.Sprintf(" %s: %5d (%4d people reporting)\n", k, countsByDate[k], len(uniquesByDate[k]))
	}

	str += fmt.Sprintf("\nDisturbance reports, counted by aircraft equipment type (where known):\n")
	for _, k := range keysByIntValDesc(countsByEquip) {
		if countsByEquip[k] < 5 {
			break
		}
		str += fmt.Sprintf(" %-40.40s: %5d\n", k, countsByEquip[k])
	}

	str += fmt.Sprintf("\nDisturbance reports, counted by Airline (where known):\n")
	for _, k := range keysByIntValDesc(countsByAirline) {
		if countsByAirline[k] < 5 || len(k) > 2 {
			continue
		}
		str += fmt.Sprintf(" %s: %6d\n", k, countsByAirline[k])
	}

	str += fmt.Sprintf("\nDisturbance reports, counted by hour of day (across all dates):\n")
	for i, n := range countsByHour {
		str += fmt.Sprintf(" %02d: %5d\n", i, n)
	}

	if countByUser {
		str += fmt.Sprintf("\nDisturbance reports, counted by user:\n")
		for _, k := range keysByIntValDesc(uniquesAll) {
			str += fmt.Sprintf(" %-60.60s: %5d\n", k, uniquesAll[k])
		}
	}

	str += fmt.Sprintf("(t=%s)\n", time.Now())

	return str, nil
}
Пример #2
0
func classbReport(c appengine.Context, s, e time.Time, opt ReportOptions) ([]ReportRow, ReportMetadata, error) {
	fdb := fdb.FlightDB{C: c}
	maybeMemcache(&fdb, e)
	tags := []string{
		flightdb.KTagReliableClassBViolation,
		flightdb.KTagLocalADSBClassBViolation,
	}

	meta := ReportMetadata{}
	h := histogram.Histogram{} // Only use it for the stats
	rows := []ReportRow{}

	seen := map[string]bool{} // Because we do two passes, we might see same flight twice.

	metars, err := metar.FetchFromNOAA(urlfetch.Client(c), "KSFO",
		s.Add(-6*time.Hour), e.Add(6*time.Hour))
	if err != nil {
		return rows, meta, err
	}

	reportFunc := func(f *flightdb.Flight) {
		if _, exists := seen[f.UniqueIdentifier()]; exists {
			return
		} else {
			seen[f.UniqueIdentifier()] = true
		}

		bestTrack := "FA"
		if f.HasTag(flightdb.KTagLocalADSBClassBViolation) {
			bestTrack = "ADSB"
		}

		_, cbt := f.SFOClassB(bestTrack, metars)

		tmpRows := []ReportRow{}

		seq := 0
		for _, cbtp := range cbt {
			if cbtp.A.IsViolation() {
				fClone := f.ShallowCopy()
				tmpRows = append(tmpRows, CBRow{seq, flight2Url(*fClone), *fClone, cbtp.TP, cbtp.A})
				seq++
			}
		}

		if len(tmpRows) == 0 {
			return
		}

		worstCBRow := tmpRows[0].(CBRow)
		if seq > 0 {
			// Select the worst row
			n, belowBy := 0, 0.0
			for i, row := range tmpRows {
				if row.(CBRow).A.BelowBy > belowBy {
					n, belowBy = i, row.(CBRow).A.BelowBy
				}
			}
			worstCBRow = tmpRows[n].(CBRow)
			worstCBRow.Seq = 0 // fake this out for the webpage
		}

		if opt.ClassB_LocalDataOnly && !f.HasTag(flightdb.KTagLocalADSBClassBViolation) {
			meta["[C] -- Skippped; not local - "+worstCBRow.TP.LongSource()]++
		} else {
			meta["[C] -- Detected via "+worstCBRow.TP.LongSource()]++
			h.Add(histogram.ScalarVal(worstCBRow.A.BelowBy))

			if opt.ClassB_OnePerFlight {
				rows = append(rows, worstCBRow)
			} else {
				rows = append(rows, tmpRows...)
			}
		}
	}

	// Need to do multiple passes, because of tagA-or-tagB sillness
	// In each case, limit to SERFR1 flights
	for _, tag := range tags {
		theseTags := []string{tag, flightdb.KTagSERFR1}
		if err := fdb.IterWith(fdb.QueryTimeRangeByTags(theseTags, s, e), reportFunc); err != nil {
			return nil, nil, err
		}
	}

	if n, err := fdb.CountTimeRangeByTags([]string{flightdb.KTagSERFR1}, s, e); err != nil {
		return nil, nil, err
	} else {
		meta["[A] Total SERFR1 Flights"] = float64(n)
	}

	if stats, valid := h.Stats(); valid {
		meta["[B] Num violating flights"] = float64(stats.N)
		meta["[D] Mean violation below Class B floor"] = float64(int(stats.Mean))
		meta["[D] Stddev"] = float64(int(stats.Stddev))
	} else {
		meta["[B] Num violating flights"] = 0.0
	}

	return rows, meta, nil
}