/* Returns intermediate stops between (end points inclusive) fromId and toId of line and direction*/
func SelectIntermediateStops(line, bound, fromId, toId string) []string {
var stopPoints []StopPoint
dbconn.AcquireLock()
err := db.Select(&stopPoints,
"SELECT naptan_id FROM line_stop_points "+
"WHERE line_id=$1 AND bound=$2 AND stop_seq >="+
"(SELECT stop_seq FROM line_stop_points WHERE "+
"line_id=$1 AND bound=$2 AND naptan_id=$3 "+
"ORDER BY stop_seq LIMIT 1) "+
"AND stop_seq <= "+
"(SELECT stop_seq FROM line_stop_points WHERE "+
"line_id=$1 AND bound=$2 AND naptan_id=$4 "+
"ORDER BY stop_seq DESC LIMIT 1) "+
"ORDER BY stop_seq ASC",
line, bound, fromId, toId)
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Panic(err)
}
if len(stopPoints) <= 1 {
logger.GetLogger().Panicf("No intermediate stops between %v and %v for line %v (%v)",
fromId, toId, line, bound)
}
result := make([]string, len(stopPoints))
for i, v := range stopPoints {
result[i] = v.NaptanId
}
logger.GetLogger().Info("Get intermediate stops (inclusive) from line_stop_points returned: %v", result)
return result
}
func mapToPerformanceTestSeries2(name string, predictedJourneyTimes, actualJourneyTimes map[time.Time]time.Duration) PerformanceTestSeries2 {
departureTimes := make([]time.Time, len(predictedJourneyTimes))
durations := make([]time.Duration, len(predictedJourneyTimes))
i := 0
/* For calculating the root mean squared error, and mean absolute error */
diffInSeconds := make([]float64, 0)
absDiffInSeconds := make([]float64, 0)
for departureTime, duration := range predictedJourneyTimes {
departureTimes[i] = departureTime
durations[i] = duration
actualJourneyTime := actualJourneyTimes[departureTime]
diffInSeconds = append(diffInSeconds, (duration - actualJourneyTime).Seconds())
absDiffInSeconds = append(absDiffInSeconds, math.Abs((duration - actualJourneyTime).Seconds()))
i++
}
rmse, err := statistics.RootMeanSquare(absDiffInSeconds)
if err != nil {
/* If lack of data points simply assume 0 */
logger.GetLogger().Error(err.Error())
rmse = 0
}
mae, err := statistics.Mean(absDiffInSeconds)
if err != nil {
/* If lack of data points simply assume 0 */
logger.GetLogger().Error(err.Error())
mae = 0
}
mad, err := statistics.Median(absDiffInSeconds)
if err != nil {
/* If lack of data points simply assume 0 */
logger.GetLogger().Error(err.Error())
mad = 0
}
variance, err := statistics.Variance(diffInSeconds)
if err != nil {
/* If lack of data points simply assume 0 */
logger.GetLogger().Error(err.Error())
variance = 0
}
return PerformanceTestSeries2{
SeriesName: name,
X: departureTimes,
Y: durations,
Count: len(durations),
RMSE: time.Duration(rmse) * time.Second,
MAE: time.Duration(mae) * time.Second,
MAD: time.Duration(mad) * time.Second,
Variance: time.Duration(variance) * time.Second,
}
}
/* Inserts into channel arrivalsBundle once data is received from tfl and parsed
map (vehicleId =>
("line" => line,
"bound" => bound,
"naptanId" => naptanId of terminus,
"expectedDeparture" => expectedDeparture in TfL format,
))
*/
func parseAllTerminusDeparturesFromBody(body []byte, numPredictionsPerStop *int) {
type Arrivals struct {
LineId string `json:"lineId"`
VehicleId string `json:"vehicleId"`
Bound string `json:"direction"`
NaptanId string `json:"naptanId"`
TimeStamp string `json:"timeStamp"`
ExpectedDeparture string `json:"expectedArrival"`
}
var arrivals []Arrivals
err := json.Unmarshal(body, &arrivals)
/* Sometimes Tfl returns "Technical error".
In such case we skip and return gracefully */
if err != nil {
logger.GetLogger().Error("Tfl failed to give proper arrivals response: %v", string(body))
return
}
/* Select all departing terminuses of all lines
map (line => (bound => naptanId)) */
interestedTerminuses := selectdb.SelectAllOrigins()
result := make(map[string]map[string]string)
for _, bus := range arrivals {
/* Check that incoming prediction is not in the past, just in case
(n.b. allow 3 min as buffer due to heavy cpu usage causing delay in processing data) */
if timeStampHasPassedBy(bus.ExpectedDeparture, 5*time.Minute) {
// Temporary fix -- sometimes webconn stops unexpectedly, works on restart of program
// Change (to a new) numPredictionsPerStop to hopefully remedy the issue
logger.GetLogger().Error("Expected arrival %v of bus %v: %v (%v) is in the past, skipped.",
bus.ExpectedDeparture, bus.VehicleId, bus.LineId, bus.Bound)
for {
newNumPredictionsPerStop := getNumPredictionsPerStop()
if *numPredictionsPerStop != newNumPredictionsPerStop {
logger.GetLogger().Info("Changed numPredictionsPerStop from %v to %v", *numPredictionsPerStop, newNumPredictionsPerStop)
*numPredictionsPerStop = newNumPredictionsPerStop
break
}
}
logger.UseNewLogFile()
return
}
/* Check if prediction refers to a departing terminus */
if bus.NaptanId == interestedTerminuses[bus.LineId][bus.Bound] {
result[bus.VehicleId] = map[string]string{
"line": bus.LineId,
"bound": bus.Bound,
"naptanId": bus.NaptanId,
"expectedDeparture": bus.ExpectedDeparture,
}
}
}
insertExpectedDeparturesIntoDB(result)
}
/* Returns the next buses that are approaching a stop given a line and bound after a certain time */
func SelectNextBuses(lineBoundCombination map[string]string, naptanId string, arrivingAfter time.Time) []NextBus {
var nextBuses []NextBus
innerQueries := make([]string, 0)
for line, bound := range lineBoundCombination {
innerQueries = append(innerQueries,
fmt.Sprintf("SELECT line, bound, naptan_id, vehicle_id, expected_arrival_time, already_departed "+
"FROM next_buses "+
"WHERE naptan_id='%v' AND line='%v' AND bound='%v' AND expected_arrival_time>=TIMESTAMP '%v'",
naptanId, line, bound, toPsqlTimeStamp(arrivingAfter)))
}
sql := fmt.Sprintf("SELECT line, bound, naptan_id, vehicle_id, expected_arrival_time, already_departed "+
"FROM (%v) AS all_next_buses ORDER BY expected_arrival_time",
strings.Join(innerQueries, " UNION ALL "))
logger.GetLogger().Info(sql)
dbconn.AcquireLock()
err := db.Select(&nextBuses, sql)
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Panic(err)
}
/* Set time zone to UTC */
for i, v := range nextBuses {
nextBuses[i].ExpectedArrivalTime = v.ExpectedArrivalTime.In(timezone.Get())
}
return nextBuses
}
/* Returns sequences of all naptanIds, grouped by line and bound
map (line => (bound => []naptanIds)) */
func SelectAllLineStopPoints() map[string]map[string][]string {
var stopPoints []StopPoint
dbconn.AcquireLock()
err := db.Select(&stopPoints,
"SELECT line_id, bound, stop_seq, naptan_id "+
"FROM line_stop_points "+
"ORDER BY line_id, bound, stop_seq")
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Panic(err)
}
result := make(map[string]map[string][]string)
for _, v := range stopPoints {
lineDetails, exists := result[v.LineId]
if !exists {
lineDetails = make(map[string][]string)
}
orderedStopPoints, exists := lineDetails[v.Bound]
if !exists {
orderedStopPoints = make([]string, 0)
}
orderedStopPoints = append(orderedStopPoints, v.NaptanId)
lineDetails[v.Bound] = orderedStopPoints
result[v.LineId] = lineDetails
}
logger.GetLogger().Info("Get existing line stop points returned: %v", result)
return result
}
/* Returns the stop sequence number, given the line, direction and naptanId of the stop
Should there be ties, an error is returned */
func SelectStopSeq(line, bound, naptanId string) (int, error) {
var stopPoints []StopPoint
dbconn.AcquireLock()
err := db.Select(&stopPoints,
"SELECT stop_seq FROM line_stop_points WHERE line_id=$1 AND bound=$2 AND naptan_id=$3",
line, bound, naptanId)
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Panic(err)
}
if len(stopPoints) == 0 {
logger.GetLogger().Error("Failed to find stop sequence number for stop %v of line %v (%v)",
naptanId, line, bound)
return -1, errors.New(
fmt.Sprintf("Failed to find stop sequence number for stop %v of line %v (%v)",
naptanId, line, bound))
}
if len(stopPoints) > 1 {
logger.GetLogger().Error("Expected unique stop sequence number for stop %v of line %v (%v), but got: %v",
naptanId, line, bound, stopPoints)
return -1, errors.New(
fmt.Sprintf("Expected unique stop sequence number for stop %v of line %v (%v), but got: %v",
naptanId, line, bound, stopPoints))
}
result := stopPoints[0].StopSeq
return result, nil
}
/* Writes to .html file for resulting performance graph */
func writePerformanceChart(wr io.Writer, charts []PerformanceChart, templatePath string) {
funcMap := template.FuncMap{
// Prettifies x and y axes into js format
"formatXAxis": func(slice []time.Time) string {
str := "['x'"
for i := 0; i < len(slice); i++ {
str += ", " + fmt.Sprintf("new Date(Date.UTC(%v, %v, %v, %v, %v, %v))",
slice[i].Year(), int(slice[i].Month())-1, slice[i].Day(),
slice[i].Hour(), slice[i].Minute(), slice[i].Second())
}
str += "]"
return str
},
"formatYAxis": func(seriesName string, slice []time.Duration) string {
str := fmt.Sprintf("['%v'", seriesName)
for i := 0; i < len(slice); i++ {
str += ", " + strconv.Itoa(int(slice[i].Minutes()))
}
str += "]"
return str
},
}
t, err := template.New(path.Base(templatePath)).Funcs(funcMap).ParseFiles(templatePath)
if err != nil {
logger.GetLogger().Panic(err)
}
err = t.Execute(wr, charts)
if err != nil {
logger.GetLogger().Panic(err)
}
}
func parseLinesFromBody(body []byte) []interface{} {
type Line struct {
Id string `json:"id"`
ModeName string `json:"modeName"`
}
var lines []Line
err := json.Unmarshal(body, &lines)
/* Sometimes Tfl returns "Technical error".
In such case we skip and return gracefully */
if err != nil {
logger.GetLogger().Panic("Tfl failed to give proper arrivals response: %v", string(body))
}
var result []interface{}
for _, line := range lines {
if line.ModeName == "bus" {
result = append(result, line.Id)
}
}
logger.GetLogger().Info("Parsed lines from tfl: %v", result)
return result
}
/* Blocks here until a bus departs from a station,
and returns the vehicleId of the bus that departed */
func blockUntilBusDeparts(line, bound, naptanId string) (string, time.Time, error) {
tflPollInterval := 50 * time.Second
departureSchedule := make(map[string]string)
site := fmt.Sprintf("https://api.tfl.gov.uk/Line/%v/Arrivals/%v?direction=%v",
line, naptanId, bound)
for _ = range time.Tick(tflPollInterval) {
var departed bool
var vehicleId string
var departureTime time.Time
var err error
departed, vehicleId, departureSchedule, departureTime, err = monitorForDeparture(site, departureSchedule, line, bound)
if err != nil {
return "", time.Time{}, err
}
if departed {
logger.GetLogger().Info("Vehicle %v of line %v (%v) departed origin %v",
vehicleId, line, bound, naptanId)
return vehicleId, departureTime, nil
}
}
logger.GetLogger().Panic("Expected indefinite blocking until departure, but failed for line %v (%v)",
line, bound)
return "", time.Time{}, nil
}
func createMedian() {
createFinalMedianSql := "CREATE OR REPLACE FUNCTION _final_median(NUMERIC[]) " +
"RETURNS NUMERIC AS " +
"$$ " +
"SELECT AVG(val) " +
"FROM ( " +
"SELECT val " +
"FROM unnest($1) val " +
"ORDER BY 1 " +
"LIMIT 2 - MOD(array_upper($1, 1), 2) " +
"OFFSET CEIL(array_upper($1, 1) / 2.0) - 1 " +
") sub; " +
"$$ " +
"LANGUAGE 'sql' IMMUTABLE"
createMedianSql := "CREATE AGGREGATE median(NUMERIC) ( " +
"SFUNC=array_append, " +
"STYPE=NUMERIC[], " +
"FINALFUNC=_final_median, " +
"INITCOND='{}' " +
")"
dbconn.AcquireLock()
tx, err := db.Begin()
tx.Exec(createFinalMedianSql)
tx.Exec(createMedianSql)
err = tx.Commit()
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Warning(err.Error())
return
}
logger.GetLogger().Info("Function 'median' created successfully")
}
func replaceTimetable(line, bound, origin string, newTimetable map[time.Weekday]map[selectdb.LocalTime]bool) {
values := make([]string, 0)
for weekday, departureTimes := range newTimetable {
for departureTime, _ := range departureTimes {
values = append(values, fmt.Sprintf("('%v', '%v', '%v', %v, TIME '%02d:%02d:00')",
line, bound, origin, int(weekday), departureTime.Hour, departureTime.Minute))
}
}
dbconn.AcquireLock()
tx := db.MustBegin()
tx.MustExec(fmt.Sprintf("DELETE FROM timetable WHERE line_id='%v' AND bound='%v' AND naptan_id='%v'", line, bound, origin))
logger.GetLogger().Info(fmt.Sprintf("INSERT INTO timetable (line_id, bound, naptan_id, weekday, departure_time) VALUES %s",
strings.Join(values, ",")))
tx.MustExec(
fmt.Sprintf("INSERT INTO timetable (line_id, bound, naptan_id, weekday, departure_time) VALUES %s",
strings.Join(values, ",")))
err := tx.Commit()
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Error(err.Error())
}
logger.GetLogger().Info("Replaced timetable for line %v (%v), stop %v", line, bound, origin)
}
func getRoutePaths(linesToPlot map[string]map[string]map[string]string) map[string][]LatLng {
routePaths := make(map[string][]LatLng)
bound := "outbound"
for line, lineDetails := range linesToPlot {
fromStop := lineDetails[bound]["fromStop"]
toStop := lineDetails[bound]["toStop"]
intermediateStops := selectdb.SelectIntermediateStops(line, bound, fromStop, toStop)
path := make([]LatLng, len(intermediateStops))
intermediateStopDetails := selectdb.SelectMultipleStops(intermediateStops)
for i, stop := range intermediateStops {
lat, err := strconv.ParseFloat(intermediateStopDetails[stop]["latitude"], 64)
if err != nil {
logger.GetLogger().Panic(err)
}
lng, err := strconv.ParseFloat(intermediateStopDetails[stop]["longitude"], 64)
if err != nil {
logger.GetLogger().Panic(err)
}
path[i] = LatLng{
Lat: lat,
Lng: lng,
}
}
routePaths[line] = path
}
return routePaths
}
/* Returns timetable for specified line, bound and stop in the form of a map
map (weekday => []("hour" => hour
"minute" => minute)) */
func SelectTimetable(line, bound, naptanId string) map[time.Weekday]map[LocalTime]bool {
var timetable []TimetableEntry
dbconn.AcquireLock()
err := db.Select(&timetable,
"SELECT weekday, departure_time FROM timetable WHERE line_id=$1 AND bound=$2 AND naptan_id=$3 ORDER BY weekday,departure_time",
line, bound, naptanId)
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Panic(err)
}
result := make(map[time.Weekday]map[LocalTime]bool)
for _, entry := range timetable {
weekday := time.Weekday(entry.Weekday)
weekdayMap, exists := result[weekday]
if !exists {
weekdayMap = make(map[LocalTime]bool, 0)
result[weekday] = weekdayMap
}
weekdayMap[LocalTime{Hour: entry.DepartureTime.Hour(), Minute: entry.DepartureTime.Minute()}] = true
}
logger.GetLogger().Info("Get timetable for line %v (%v), stop %v returned: %v", line, bound, naptanId, result)
return result
}
/* To handle the situation when stops logged from Tfl are not consecutive,
i.e. fromStop and toStop are > 1 stop apart
Applies interpolation to estimate neighbouring journey times, and
updates the values for insertion into database
@See interpolateTimes for implementation details */
func interpolateStops(line, bound, vehicleId, fromStop, toStop, oldTimeStamp, newTimeStamp, oldTflTimeStamp, newTflTimeStamp string,
values, skippedStopsValues, rawArrivalTimes *[]string, cachedEntry map[string]string) {
// Discard if interpolating too many stops (>= 10 stops)
interpolationThreshold := 10
// Find intermediate stops
intermediateStops := selectdb.SelectIntermediateStops(line, bound, fromStop, toStop)
intermediateStopsLength := len(intermediateStops)
if intermediateStopsLength <= 2 {
logger.GetLogger().Panicf("Expected at least one intermediate stop for line %v (%v), between %v and %v. But got none.",
line, bound, fromStop, toStop)
}
if intermediateStopsLength >= interpolationThreshold {
logger.GetLogger().Error("Bus %v of line %v (%v) skipped too many stops (%v stops), between %v and %v. Aborting interpolation.",
vehicleId, line, bound, intermediateStopsLength, fromStop, toStop)
return
}
/* Find out interpolated times */
interpolatedTimes, err := interpolateTimes(line, bound, vehicleId, intermediateStops, oldTimeStamp, newTimeStamp, cachedEntry, rawArrivalTimes)
if err != nil {
logger.GetLogger().Error(err.Error())
return
}
for i := 0; i < intermediateStopsLength-1; i++ {
if interpolatedTimes[i].IsZero() || interpolatedTimes[i+1].IsZero() {
// Check that record is not invalidated
continue
}
timeDiff := int(interpolatedTimes[i+1].Sub(interpolatedTimes[i]).Seconds())
*values = append(*values, fmt.Sprintf("('%v', '%v', '%v', '%v', TIMESTAMP '%v', TIMESTAMP '%v', "+
"'%v', TIMESTAMP '%v', TIMESTAMP '%v', INTERVAL '%v seconds', TRUE)",
line, bound, vehicleId,
intermediateStops[i], timeToPsqlTimeStamp(interpolatedTimes[i]), toPsqlTimeStamp(oldTflTimeStamp),
intermediateStops[i+1], timeToPsqlTimeStamp(interpolatedTimes[i+1]), toPsqlTimeStamp(newTflTimeStamp),
timeDiff))
}
// Strip end points away from intermediate points before inserting into skipped_stops_history
intermediateStopsEndPointsExclusive := intermediateStops[1 : intermediateStopsLength-1]
for i, v := range intermediateStopsEndPointsExclusive {
intermediateStopsEndPointsExclusive[i] = "'" + v + "'"
}
*skippedStopsValues = append(*skippedStopsValues,
fmt.Sprintf("('%v', '%v', '%v', '%v', TIMESTAMP '%v', TIMESTAMP '%v', '%v', TIMESTAMP '%v', TIMESTAMP '%v', ARRAY[%v], %v)",
line, bound, vehicleId,
intermediateStops[0], timeToPsqlTimeStamp(interpolatedTimes[0]), toPsqlTimeStamp(oldTflTimeStamp),
intermediateStops[intermediateStopsLength-1], timeToPsqlTimeStamp(interpolatedTimes[intermediateStopsLength-1]), toPsqlTimeStamp(newTflTimeStamp),
strings.Join(intermediateStopsEndPointsExclusive, ","), len(intermediateStopsEndPointsExclusive)))
}
func parseLineStopPointsFromBody(body []byte) ([]string, map[string]map[string]string) {
type StopPoint struct {
Id string `json:"id"`
IcsId string `json:"icsId"`
StationId string `json:"stationId"`
Name string `json:"name"`
Latitude float64 `json:"lat"`
Longitude float64 `json:"lon"`
}
type StopPointSequences struct {
StopPt []StopPoint `json:"stopPoint"`
}
type Line struct {
IsOutboundOnly bool `json:"isOutboundOnly"`
LineId string `json:"lineId"`
Direction string `json:"direction"`
StopSequence []StopPointSequences `json:"stopPointSequences"`
}
var line Line
err := json.Unmarshal(body, &line)
if err != nil {
logger.GetLogger().Panicf("Tfl failed to give proper body: %v", string(body))
}
if line.IsOutboundOnly && line.Direction == "inbound" {
logger.GetLogger().Info("Line %v is outbound only", line.LineId)
return nil, nil
}
if len(line.StopSequence) == 0 {
logger.GetLogger().Error("Failed to parse stop sequence for line %v (%v)", line.LineId, line.Direction)
return nil, nil
}
stopPoints := line.StopSequence[0].StopPt
result := make([]string, len(stopPoints))
/* Get also stop info including name, icscode, lat, lng */
stopInfos := make(map[string]map[string]string)
for i, stopPoint := range stopPoints {
result[i] = stopPoint.Id
stopInfos[stopPoint.Id] = map[string]string{
"name": stopPoint.Name,
"icscode": stopPoint.IcsId,
"latitude": strconv.FormatFloat(stopPoint.Latitude, 'f', -1, 64),
"longitude": strconv.FormatFloat(stopPoint.Longitude, 'f', -1, 64),
"stationNaptan": stopPoint.StationId,
}
}
logger.GetLogger().Info("Parsed line %v (%v) for stops: %v", line.LineId, line.Direction, result)
return result, stopInfos
}
func getStopInfos(lineStopPointsByRoute map[string]map[string][]string) map[string]map[string][]TflStopPoint {
/* Get slice of all stops */
stopsMap := make(map[string]bool)
for _, lineDetails := range lineStopPointsByRoute {
for _, orderedStopPoints := range lineDetails {
for _, stop := range orderedStopPoints {
stopsMap[stop] = true
}
}
}
stops := make([]string, len(stopsMap))
counter := 0
for stop, _ := range stopsMap {
stops[counter] = stop
counter++
}
stopInfos := selectdb.SelectMultipleStops(stops)
result := make(map[string]map[string][]TflStopPoint)
for line, lineDetails := range lineStopPointsByRoute {
for bound, orderedStopPoints := range lineDetails {
for _, naptanId := range orderedStopPoints {
_, exists := result[line]
if !exists {
result[line] = make(map[string][]TflStopPoint)
}
_, exists = result[line][bound]
if !exists {
result[line][bound] = make([]TflStopPoint, 0)
}
stopInfo := stopInfos[naptanId]
name := stopInfo["name"]
latitude, err := strconv.ParseFloat(stopInfo["latitude"], 64)
if err != nil {
logger.GetLogger().Panic(err)
}
longitude, err := strconv.ParseFloat(stopInfo["longitude"], 64)
if err != nil {
logger.GetLogger().Panic(err)
}
result[line][bound] = append(result[line][bound],
TflStopPoint{
Name: name,
NaptanId: naptanId,
Latitude: latitude,
Longitude: longitude,
})
}
}
}
return result
}
/* Returns the tfl predictions at the times specified for a given line, bound, origin and destination
map time.Time => time.Duration */
func SelectTflPredictions(line, bound, fromStop, toStop string, times []time.Time) map[time.Time]time.Duration {
tflPredictions := make([]int64, len(times))
dbconn.AcquireLock()
tx, err := db.Begin()
for i, targetTime := range times {
upperBound := targetTime.Add(10 * time.Minute)
lowerBound := targetTime.Add(-10 * time.Minute)
logger.GetLogger().Info(fmt.Sprintf("SELECT EXTRACT(epoch FROM predicted_time) FROM ("+
"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time>='%v' AND departure_time <= '%v' ORDER BY departure_time LIMIT 1) "+
"UNION ALL "+
"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time<'%v' AND departure_time >= '%v' ORDER BY departure_time DESC LIMIT 1) "+
") AS result ORDER BY abs(EXTRACT(epoch FROM departure_time-'%v')) LIMIT 1",
line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(upperBound),
line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(lowerBound),
toPsqlTimeStamp(targetTime)))
err = tx.QueryRow(
fmt.Sprintf("SELECT EXTRACT(epoch FROM predicted_time) FROM ("+
"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time>='%v' AND departure_time <= '%v' ORDER BY departure_time LIMIT 1) "+
"UNION ALL "+
"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time<'%v' AND departure_time >= '%v' ORDER BY departure_time DESC LIMIT 1) "+
") AS result ORDER BY abs(EXTRACT(epoch FROM departure_time-'%v')) LIMIT 1",
line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(upperBound),
line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(lowerBound),
toPsqlTimeStamp(targetTime))).Scan(&tflPredictions[i])
if err != nil {
logger.GetLogger().Error(err.Error())
tflPredictions[i] = 0
}
}
err = tx.Commit()
dbconn.ReleaseLock()
if err != nil {
logger.GetLogger().Panic(err)
}
result := make(map[time.Time]time.Duration)
for i, t := range times {
/* Skip if no recent prediction found (within 20 minute window) */
tflPrediction := tflPredictions[i]
if tflPrediction == 0 {
continue
}
result[t] = time.Duration(tflPredictions[i]) * time.Second
}
return result
}
func parseCumulativeJourneyTimesFromBody(line, bound string, body []byte) []map[string]int {
type Interval struct {
StopId string `json:"stopId"`
TimeToArrival float32 `json:"timeToArrival"`
}
type StationInterval struct {
Intervals []Interval `json:"intervals"`
}
type Route struct {
StationIntervals []StationInterval `json:"stationIntervals"`
}
type Times struct {
Routes []Route `json:"routes"`
}
type Timetable struct {
Timetable Times `json:"timetable"`
}
var timetable Timetable
err := json.Unmarshal(body, &timetable)
/* Sometimes Tfl returns "Technical error".
In such case we skip and return gracefully */
if err != nil {
logger.GetLogger().Error("Tfl failed to give proper arrivals response: %v", string(body))
return make([]map[string]int, 0)
}
routes := timetable.Timetable.Routes
if len(routes) == 0 {
logger.GetLogger().Error("Failed to parse routes in timetable for line %v (%v)", line, bound)
return make([]map[string]int, 0)
}
stationIntervals := routes[0].StationIntervals
if len(stationIntervals) == 0 {
logger.GetLogger().Error("Failed to parse station intervals in timetable for line %v (%v)", line, bound)
return make([]map[string]int, 0)
}
intervals := stationIntervals[0].Intervals
var result = make([]map[string]int, len(intervals))
for i, interval := range intervals {
m := make(map[string]int)
m[interval.StopId] = int(interval.TimeToArrival)
result[i] = m
}
logger.GetLogger().Info("Parsed cumulative journey times for line %v (%v): %v", line, bound, result)
return result
}
/* Returns current, historic, and real time durations */
func getRouteDuration(line, bound string, orderedStopPoints []string,
currentEstimates, currentHistoricEstimates, currentRealTimeEstimates map[string]map[string]time.Duration,
delayCounts map[string]map[string]int) (time.Duration, time.Duration, time.Duration, error) {
duration := time.Duration(0)
historicDuration := time.Duration(0)
realTimeDuration := time.Duration(0)
for i := 0; i < len(orderedStopPoints)-1; i++ {
fromStop := orderedStopPoints[i]
toStop := orderedStopPoints[i+1]
historicEstimate := time.Duration(0)
realTimeEstimate := time.Duration(0)
_, exists := currentHistoricEstimates[fromStop]
if !exists {
logger.GetLogger().Error("Lack current historic estimates for line %v (%v) fromStop: %v", line, bound, fromStop)
} else {
historicEstimate, exists = currentHistoricEstimates[fromStop][toStop]
if !exists {
logger.GetLogger().Error("Lack current historic estimates for line %v (%v) fromStop: %v, toStop: %v", line, bound, fromStop, toStop)
}
}
_, exists = currentRealTimeEstimates[fromStop]
if !exists {
logger.GetLogger().Error("Lack current real time estimates for line %v (%v) fromStop: %v", line, bound, fromStop)
} else {
realTimeEstimate, exists = currentRealTimeEstimates[fromStop][toStop]
if !exists {
logger.GetLogger().Error("Lack current real time estimates for line %v (%v) fromStop: %v, toStop: %v", line, bound, fromStop, toStop)
}
}
if historicEstimate != 0 {
historicDuration += historicEstimate
} else {
historicDuration += realTimeEstimate
}
if realTimeEstimate != 0 {
realTimeDuration += realTimeEstimate
} else {
realTimeDuration += historicEstimate
}
/* Use delay if trustworthy */
if delayCounts[fromStop][toStop] >= 3 {
duration += realTimeEstimate
} else {
duration += historicEstimate
}
}
return duration, historicDuration, realTimeDuration, nil
}
/* Writes to .html file for resulting bus route map */
func writeBusMap(wr io.Writer, busMap BusMap, templatePath string) {
t, err := template.New(path.Base(templatePath)).ParseFiles(templatePath)
if err != nil {
logger.GetLogger().Panic(err)
}
err = t.Execute(wr, busMap)
if err != nil {
logger.GetLogger().Panic(err)
}
}
/* Returns map (vehicleId => (
"direction" => bound, "naptanId" => naptanId,
"timeStamp" => timeStamp, "expectedArrival" => expectedArrival))
for each bus tracked */
func parseArrivalsFromBody(body []byte) map[string]map[string]string {
type Arrivals struct {
VehicleId string `json:"vehicleId"`
Direction string `json:"direction"`
NaptanId string `json:"naptanId"`
TimeStamp string `json:"timeStamp"`
TimeToStation int `json:"timeToStation"`
ExpectedArrival string `json:"expectedArrival"`
}
var arrivals []Arrivals
err := json.Unmarshal(body, &arrivals)
/* Sometimes Tfl returns "Technical error".
In such case we skip and return gracefully */
if err != nil {
logger.GetLogger().Error("Tfl failed to give proper arrivals response: %v", string(body))
return make(map[string]map[string]string)
}
result := make(map[string]map[string]string)
for _, bus := range arrivals {
bundle, bundleExists := result[bus.VehicleId]
if !bundleExists {
bundle = make(map[string]string)
}
/* Tfl api seems to return all future stops for a vehicle.
Ensures entry with lowest timeToStation is recorded in the map, thus pointing to the next stop
CATCH: Make sure timeToStation is not negative, i.e. a prediction persists even though it is
now a past event */
timeToStation, recordAlreadyExists := bundle["timeToStation"]
if recordAlreadyExists {
recordedTimeToStation, _ := strconv.Atoi(timeToStation)
if recordedTimeToStation < bus.TimeToStation {
insertExpectedArrivalToBundle(bundle, bus.NaptanId, bus.ExpectedArrival)
result[bus.VehicleId] = bundle
continue
}
}
bundle["direction"] = bus.Direction
bundle["naptanId"] = bus.NaptanId
bundle["timeStamp"] = bus.TimeStamp
bundle["timeToStation"] = strconv.Itoa(bus.TimeToStation)
bundle["expectedArrival"] = bus.ExpectedArrival
insertExpectedArrivalToBundle(bundle, bus.NaptanId, bus.ExpectedArrival)
result[bus.VehicleId] = bundle
}
logger.GetLogger().Info("Parsed arrivals from tfl: %v", result)
return result
}
/* Plots the journey times throughout a day between a pair of stops */
func JourneyTimePlot(adjacentStops map[string]string, dates []time.Time,
interval time.Duration, wr io.Writer, templatePath string) {
var charts []JourneyTimeChart
for fromStop, toStop := range adjacentStops {
logger.GetLogger().Info("Handling journey: %v -> %v", fromStop, toStop)
series := make([]JourneyTimeSeries, 0)
var partitionedTimes []Partition
for _, date := range dates {
// Partition between startTime (start of day) and endTime (end of day)
startTime := date.Truncate(24 * time.Hour)
endTime := date.Truncate(24 * time.Hour).Add(24 * time.Hour)
partitionedTimes = getPartitionedTimes(startTime, endTime, interval)
// Get average journey times corresponding to partitions, and their 95% confidence intervals
partitionedAvgJourneyTimes, stdDevJourneyTimes, invalidIndices := getAverageJourneyTimes(fromStop, toStop, partitionedTimes)
lowerCI, upperCI, err := statistics.ConfidenceInterval(partitionedAvgJourneyTimes, stdDevJourneyTimes)
if err != nil {
logger.GetLogger().Panic(err)
}
// Insert into series the journey time of each partition to plot chart
series = append(series, JourneyTimeSeries{date, partitionedAvgJourneyTimes, lowerCI, upperCI, invalidIndices,
randomColour()})
}
// Generate chart - Write to *.html
xlabs := make([]string, len(partitionedTimes))
for i := 0; i < len(partitionedTimes); i++ {
start := partitionedTimes[i].start
// e.g. label x axis with time - e.g. 3:00 PM
xlabs[i] = start.Format(time.Kitchen)
}
// Title of chart
fromStopName := selectdb.SelectStop(fromStop)
toStopName := selectdb.SelectStop(toStop)
title := fmt.Sprintf("Journey times between %v and %v", fromStopName, toStopName)
chart := JourneyTimeChart{
title,
fromStop,
toStop,
xlabs,
series,
}
charts = append(charts, chart)
}
writeJourneyTimeCharts(wr, charts, templatePath)
}
func listFiles(dir string) []string {
d, err := os.Open(dir)
if err != nil {
logger.GetLogger().Panic(err)
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
logger.GetLogger().Panic(err)
}
return names
}
/* Returns map (vehicleId => (
"direction" => bound, "naptanId" => naptanId,
"timeStamp" => timeStamp, "expectedArrival" => expectedArrival))
for each bus tracked */
func parseStreamArrivalsFromResponse(response *http.Response) <-chan map[string]map[string]string {
out := make(chan map[string]map[string]string)
go func() {
var predictionArray []json.Number
dec := json.NewDecoder(response.Body)
for dec.More() {
err := dec.Decode(&predictionArray)
if err != nil {
logger.GetLogger().Panic(err)
}
if len(predictionArray) == 0 {
logger.GetLogger().Panic("Unexpected empty prediction array")
}
responseType, _ := predictionArray[0].Int64()
switch responseType {
/* Response type 1 corresponds to prediction array.
Parse prediction array */
case 1:
if len(predictionArray) != 6 {
logger.GetLogger().Info("Expected prediction array to have length 5, but got %v. Prediction array: %v",
len(predictionArray), predictionArray)
}
naptanId := predictionArray[1].String()
visitNumber := predictionArray[2].String()
directionId, _ := predictionArray[3].Int64()
vehicleId := predictionArray[4].String()
expectedArrivalUnixEpoch, _ := predictionArray[5].Int64()
result := make(map[string]map[string]string)
bundle := make(map[string]string)
bundle["naptanId"] = naptanId
bundle["visitNumber"] = visitNumber
bundle["direction"] = directionIdToBound(directionId, predictionArray)
bundle["timeStamp"] = toTflTimeStamp(time.Now().In(timezone.Get()))
bundle["expectedArrival"] = toTflTimeStamp(time.Unix(0, expectedArrivalUnixEpoch))
result[vehicleId] = bundle
out <- result
default:
logger.GetLogger().Error("Encountered non-prediction array: %v", predictionArray)
continue
}
}
}()
return out
}
func main() {
lines := selectdb.SelectLines()
ch := make(chan string)
for _, line_id := range lines {
go func(line string) {
updatedb.UpdateAdjacentStops(line)
ch <- line
}(line_id)
}
for i := 0; i < len(lines); i++ {
logger.GetLogger().Info("Done for line %v", <-ch)
}
logger.GetLogger().Info("Adjacent stops for all lines updated")
}
func parseTflEstimateFromBody(line string, body []byte) time.Duration {
type TflLineIdentifier struct {
Id string `json:"id"`
Bound string `json:"bound"`
}
type TflRouteOption struct {
Directions []string `json:"directions"`
LineIdentifier TflLineIdentifier `json:"lineIdentifier"`
}
type TflLeg struct {
Duration int `json:"duration"`
RouteOptions []TflRouteOption `json:"routeOptions"`
}
type TflJourney struct {
TflDuration int `json:"duration"`
Legs []TflLeg `json:"legs"`
}
type TflDirections struct {
Journeys []TflJourney `json:"journeys"`
}
var directions TflDirections
err := json.Unmarshal(body, &directions)
if err != nil {
logger.GetLogger().Panicf("Tfl failed to give proper arrivals response: %v", string(body))
}
journeys := directions.Journeys
if len(journeys) == 0 {
logger.GetLogger().Error("Lack of journeys for route: %v", line)
return 0
}
for _, journey := range journeys {
for _, leg := range journey.Legs {
for _, routeOption := range leg.RouteOptions {
if routeOption.LineIdentifier.Id != line {
continue
}
}
}
return time.Duration(journey.TflDuration) * time.Minute
}
return 0
}
/* Inserts credentials into site url request */
func GetResponseFromTfl(site string) *http.Response {
site = fmt.Sprintf("%v&app_id=%v&app_key=%v", site,
credentials.GetAppId(), credentials.GetAppKey())
queryLimitChannel <- 0
logger.GetLogger().Info("Attempting to access tfl site: %s", site)
response, err := http.Get(site)
if err != nil {
logger.GetLogger().Panic(err)
}
logger.GetLogger().Info("Response from tfl status code: %v", response.StatusCode)
return response
}
func getGoogleMapCredentials() GoogleMapCredentials {
googleMapCredentialsFile, err := os.Open(googleMapCredentialsPath)
if err != nil {
logger.GetLogger().Panic(err)
}
defer googleMapCredentialsFile.Close()
var credentials GoogleMapCredentials
err = json.NewDecoder(googleMapCredentialsFile).Decode(&credentials)
if err != nil {
logger.GetLogger().Panic(err)
}
logger.GetLogger().Info("Go server credentials decoded - key: %v", credentials.Key)
return credentials
}
func getTflCredentials() TflCredentials {
tflCredentialsFile, err := os.Open(tflCredentialsPath)
if err != nil {
logger.GetLogger().Panic(err)
}
defer tflCredentialsFile.Close()
var credentials TflCredentials
err = json.NewDecoder(tflCredentialsFile).Decode(&credentials)
if err != nil {
logger.GetLogger().Panic(err)
}
logger.GetLogger().Info("Tfl credentials decoded - app_id: %v, app_key: %v", credentials.App_id, credentials.App_key)
return credentials
}
func getTflStreamCredentials() TflStreamCredentials {
tflStreamCredentialsFile, err := os.Open(tflStreamCredentialsPath)
if err != nil {
logger.GetLogger().Panic(err)
}
defer tflStreamCredentialsFile.Close()
var credentials TflStreamCredentials
err = json.NewDecoder(tflStreamCredentialsFile).Decode(&credentials)
if err != nil {
logger.GetLogger().Panic(err)
}
logger.GetLogger().Info("Tfl stream credentials decoded - username: %v, password: %v", credentials.Username, credentials.Password)
return credentials
}
