/* Returns intermediate stops between (end points inclusive) fromId and toId of line and direction*/
func SelectIntermediateStops(line, bound, fromId, toId string) []string {
	var stopPoints []StopPoint

	dbconn.AcquireLock()
	err := db.Select(&stopPoints,
		"SELECT naptan_id FROM line_stop_points "+
			"WHERE line_id=$1 AND bound=$2 AND stop_seq >="+
			"(SELECT stop_seq FROM line_stop_points WHERE "+
			"line_id=$1 AND bound=$2 AND naptan_id=$3 "+
			"ORDER BY stop_seq LIMIT 1) "+
			"AND stop_seq <= "+
			"(SELECT stop_seq FROM line_stop_points WHERE "+
			"line_id=$1 AND bound=$2 AND naptan_id=$4 "+
			"ORDER BY stop_seq DESC LIMIT 1) "+
			"ORDER BY stop_seq ASC",
		line, bound, fromId, toId)
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Panic(err)
	}

	if len(stopPoints) <= 1 {
		logger.GetLogger().Panicf("No intermediate stops between %v and %v for line %v (%v)",
			fromId, toId, line, bound)
	}

	result := make([]string, len(stopPoints))
	for i, v := range stopPoints {
		result[i] = v.NaptanId
	}
	logger.GetLogger().Info("Get intermediate stops (inclusive) from line_stop_points returned: %v", result)
	return result
}
func mapToPerformanceTestSeries2(name string, predictedJourneyTimes, actualJourneyTimes map[time.Time]time.Duration) PerformanceTestSeries2 {

	departureTimes := make([]time.Time, len(predictedJourneyTimes))
	durations := make([]time.Duration, len(predictedJourneyTimes))
	i := 0

	/* For calculating the root mean squared error, and mean absolute error */
	diffInSeconds := make([]float64, 0)
	absDiffInSeconds := make([]float64, 0)

	for departureTime, duration := range predictedJourneyTimes {
		departureTimes[i] = departureTime
		durations[i] = duration

		actualJourneyTime := actualJourneyTimes[departureTime]
		diffInSeconds = append(diffInSeconds, (duration - actualJourneyTime).Seconds())
		absDiffInSeconds = append(absDiffInSeconds, math.Abs((duration - actualJourneyTime).Seconds()))

		i++
	}

	rmse, err := statistics.RootMeanSquare(absDiffInSeconds)
	if err != nil {
		/* If lack of data points simply assume 0 */
		logger.GetLogger().Error(err.Error())
		rmse = 0
	}

	mae, err := statistics.Mean(absDiffInSeconds)
	if err != nil {
		/* If lack of data points simply assume 0 */
		logger.GetLogger().Error(err.Error())
		mae = 0
	}

	mad, err := statistics.Median(absDiffInSeconds)
	if err != nil {
		/* If lack of data points simply assume 0 */
		logger.GetLogger().Error(err.Error())
		mad = 0
	}

	variance, err := statistics.Variance(diffInSeconds)
	if err != nil {
		/* If lack of data points simply assume 0 */
		logger.GetLogger().Error(err.Error())
		variance = 0
	}

	return PerformanceTestSeries2{
		SeriesName: name,
		X:          departureTimes,
		Y:          durations,
		Count:      len(durations),
		RMSE:       time.Duration(rmse) * time.Second,
		MAE:        time.Duration(mae) * time.Second,
		MAD:        time.Duration(mad) * time.Second,
		Variance:   time.Duration(variance) * time.Second,
	}
}
/* Inserts into channel arrivalsBundle once data is received from tfl and parsed
  map (vehicleId =>
	("line" => line,
	 "bound" => bound,
	 "naptanId" => naptanId of terminus,
	 "expectedDeparture" => expectedDeparture in TfL format,
	))
*/
func parseAllTerminusDeparturesFromBody(body []byte, numPredictionsPerStop *int) {

	type Arrivals struct {
		LineId            string `json:"lineId"`
		VehicleId         string `json:"vehicleId"`
		Bound             string `json:"direction"`
		NaptanId          string `json:"naptanId"`
		TimeStamp         string `json:"timeStamp"`
		ExpectedDeparture string `json:"expectedArrival"`
	}

	var arrivals []Arrivals

	err := json.Unmarshal(body, &arrivals)

	/* Sometimes Tfl returns "Technical error".
	   In such case we skip and return gracefully */
	if err != nil {
		logger.GetLogger().Error("Tfl failed to give proper arrivals response: %v", string(body))
		return
	}

	/* Select all departing terminuses of all lines
	   map (line => (bound => naptanId)) */
	interestedTerminuses := selectdb.SelectAllOrigins()

	result := make(map[string]map[string]string)
	for _, bus := range arrivals {

		/* Check that incoming prediction is not in the past, just in case
		   (n.b. allow 3 min as buffer due to heavy cpu usage causing delay in processing data)  */
		if timeStampHasPassedBy(bus.ExpectedDeparture, 5*time.Minute) {
			// Temporary fix -- sometimes webconn stops unexpectedly, works on restart of program
			// Change (to a new) numPredictionsPerStop to hopefully remedy the issue
			logger.GetLogger().Error("Expected arrival %v of bus %v: %v (%v) is in the past, skipped.",
				bus.ExpectedDeparture, bus.VehicleId, bus.LineId, bus.Bound)
			for {
				newNumPredictionsPerStop := getNumPredictionsPerStop()
				if *numPredictionsPerStop != newNumPredictionsPerStop {
					logger.GetLogger().Info("Changed numPredictionsPerStop from %v to %v", *numPredictionsPerStop, newNumPredictionsPerStop)
					*numPredictionsPerStop = newNumPredictionsPerStop
					break
				}
			}
			logger.UseNewLogFile()
			return
		}

		/* Check if prediction refers to a departing terminus */
		if bus.NaptanId == interestedTerminuses[bus.LineId][bus.Bound] {
			result[bus.VehicleId] = map[string]string{
				"line":              bus.LineId,
				"bound":             bus.Bound,
				"naptanId":          bus.NaptanId,
				"expectedDeparture": bus.ExpectedDeparture,
			}
		}
	}
	insertExpectedDeparturesIntoDB(result)
}
/* Returns the next buses that are approaching a stop given a line and bound after a certain time */
func SelectNextBuses(lineBoundCombination map[string]string, naptanId string, arrivingAfter time.Time) []NextBus {
	var nextBuses []NextBus

	innerQueries := make([]string, 0)

	for line, bound := range lineBoundCombination {
		innerQueries = append(innerQueries,
			fmt.Sprintf("SELECT line, bound, naptan_id, vehicle_id, expected_arrival_time, already_departed "+
				"FROM next_buses "+
				"WHERE naptan_id='%v' AND line='%v' AND bound='%v' AND expected_arrival_time>=TIMESTAMP '%v'",
				naptanId, line, bound, toPsqlTimeStamp(arrivingAfter)))
	}

	sql := fmt.Sprintf("SELECT line, bound, naptan_id, vehicle_id, expected_arrival_time, already_departed "+
		"FROM (%v) AS all_next_buses ORDER BY expected_arrival_time",
		strings.Join(innerQueries, " UNION ALL "))

	logger.GetLogger().Info(sql)
	dbconn.AcquireLock()
	err := db.Select(&nextBuses, sql)
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Panic(err)
	}

	/* Set time zone to UTC */
	for i, v := range nextBuses {
		nextBuses[i].ExpectedArrivalTime = v.ExpectedArrivalTime.In(timezone.Get())
	}
	return nextBuses
}
/* Returns sequences of all naptanIds, grouped by line and bound
   map (line => (bound => []naptanIds)) */
func SelectAllLineStopPoints() map[string]map[string][]string {
	var stopPoints []StopPoint

	dbconn.AcquireLock()
	err := db.Select(&stopPoints,
		"SELECT line_id, bound, stop_seq, naptan_id "+
			"FROM line_stop_points "+
			"ORDER BY line_id, bound, stop_seq")
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Panic(err)
	}

	result := make(map[string]map[string][]string)
	for _, v := range stopPoints {
		lineDetails, exists := result[v.LineId]
		if !exists {
			lineDetails = make(map[string][]string)
		}

		orderedStopPoints, exists := lineDetails[v.Bound]
		if !exists {
			orderedStopPoints = make([]string, 0)
		}

		orderedStopPoints = append(orderedStopPoints, v.NaptanId)
		lineDetails[v.Bound] = orderedStopPoints
		result[v.LineId] = lineDetails
	}
	logger.GetLogger().Info("Get existing line stop points returned: %v", result)
	return result
}
/* Returns the stop sequence number, given the line, direction and naptanId of the stop
   Should there be ties, an error is returned */
func SelectStopSeq(line, bound, naptanId string) (int, error) {
	var stopPoints []StopPoint

	dbconn.AcquireLock()
	err := db.Select(&stopPoints,
		"SELECT stop_seq FROM line_stop_points WHERE line_id=$1 AND bound=$2 AND naptan_id=$3",
		line, bound, naptanId)
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Panic(err)
	}

	if len(stopPoints) == 0 {
		logger.GetLogger().Error("Failed to find stop sequence number for stop %v of line %v (%v)",
			naptanId, line, bound)
		return -1, errors.New(
			fmt.Sprintf("Failed to find stop sequence number for stop %v of line %v (%v)",
				naptanId, line, bound))
	}

	if len(stopPoints) > 1 {
		logger.GetLogger().Error("Expected unique stop sequence number for stop %v of line %v (%v), but got: %v",
			naptanId, line, bound, stopPoints)
		return -1, errors.New(
			fmt.Sprintf("Expected unique stop sequence number for stop %v of line %v (%v), but got: %v",
				naptanId, line, bound, stopPoints))
	}

	result := stopPoints[0].StopSeq
	return result, nil
}
/* Writes to .html file for resulting performance graph */
func writePerformanceChart(wr io.Writer, charts []PerformanceChart, templatePath string) {

	funcMap := template.FuncMap{
		// Prettifies x and y axes into js format
		"formatXAxis": func(slice []time.Time) string {
			str := "['x'"
			for i := 0; i < len(slice); i++ {
				str += ", " + fmt.Sprintf("new Date(Date.UTC(%v, %v, %v, %v, %v, %v))",
					slice[i].Year(), int(slice[i].Month())-1, slice[i].Day(),
					slice[i].Hour(), slice[i].Minute(), slice[i].Second())
			}
			str += "]"
			return str
		},
		"formatYAxis": func(seriesName string, slice []time.Duration) string {
			str := fmt.Sprintf("['%v'", seriesName)
			for i := 0; i < len(slice); i++ {
				str += ", " + strconv.Itoa(int(slice[i].Minutes()))
			}
			str += "]"
			return str
		},
	}

	t, err := template.New(path.Base(templatePath)).Funcs(funcMap).ParseFiles(templatePath)
	if err != nil {
		logger.GetLogger().Panic(err)
	}

	err = t.Execute(wr, charts)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
}
func parseLinesFromBody(body []byte) []interface{} {
	type Line struct {
		Id       string `json:"id"`
		ModeName string `json:"modeName"`
	}

	var lines []Line

	err := json.Unmarshal(body, &lines)

	/* Sometimes Tfl returns "Technical error".
	   In such case we skip and return gracefully */
	if err != nil {
		logger.GetLogger().Panic("Tfl failed to give proper arrivals response: %v", string(body))
	}

	var result []interface{}
	for _, line := range lines {
		if line.ModeName == "bus" {
			result = append(result, line.Id)
		}
	}

	logger.GetLogger().Info("Parsed lines from tfl: %v", result)
	return result
}
/* Blocks here until a bus departs from a station,
   and returns the vehicleId of the bus that departed */
func blockUntilBusDeparts(line, bound, naptanId string) (string, time.Time, error) {
	tflPollInterval := 50 * time.Second

	departureSchedule := make(map[string]string)

	site := fmt.Sprintf("https://api.tfl.gov.uk/Line/%v/Arrivals/%v?direction=%v",
		line, naptanId, bound)

	for _ = range time.Tick(tflPollInterval) {
		var departed bool
		var vehicleId string
		var departureTime time.Time
		var err error

		departed, vehicleId, departureSchedule, departureTime, err = monitorForDeparture(site, departureSchedule, line, bound)
		if err != nil {
			return "", time.Time{}, err
		}
		if departed {
			logger.GetLogger().Info("Vehicle %v of line %v (%v) departed origin %v",
				vehicleId, line, bound, naptanId)
			return vehicleId, departureTime, nil
		}
	}
	logger.GetLogger().Panic("Expected indefinite blocking until departure, but failed for line %v (%v)",
		line, bound)
	return "", time.Time{}, nil
}
func createMedian() {
	createFinalMedianSql := "CREATE OR REPLACE FUNCTION _final_median(NUMERIC[]) " +
		"RETURNS NUMERIC AS " +
		"$$ " +
		"SELECT AVG(val) " +
		"FROM ( " +
		"SELECT val " +
		"FROM unnest($1) val " +
		"ORDER BY 1 " +
		"LIMIT  2 - MOD(array_upper($1, 1), 2) " +
		"OFFSET CEIL(array_upper($1, 1) / 2.0) - 1 " +
		") sub; " +
		"$$ " +
		"LANGUAGE 'sql' IMMUTABLE"
	createMedianSql := "CREATE AGGREGATE median(NUMERIC) ( " +
		"SFUNC=array_append, " +
		"STYPE=NUMERIC[], " +
		"FINALFUNC=_final_median, " +
		"INITCOND='{}' " +
		")"

	dbconn.AcquireLock()
	tx, err := db.Begin()
	tx.Exec(createFinalMedianSql)
	tx.Exec(createMedianSql)
	err = tx.Commit()
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Warning(err.Error())
		return
	}
	logger.GetLogger().Info("Function 'median' created successfully")
}
func replaceTimetable(line, bound, origin string, newTimetable map[time.Weekday]map[selectdb.LocalTime]bool) {
	values := make([]string, 0)
	for weekday, departureTimes := range newTimetable {
		for departureTime, _ := range departureTimes {
			values = append(values, fmt.Sprintf("('%v', '%v', '%v', %v, TIME '%02d:%02d:00')",
				line, bound, origin, int(weekday), departureTime.Hour, departureTime.Minute))
		}
	}

	dbconn.AcquireLock()
	tx := db.MustBegin()
	tx.MustExec(fmt.Sprintf("DELETE FROM timetable WHERE line_id='%v' AND bound='%v' AND naptan_id='%v'", line, bound, origin))

	logger.GetLogger().Info(fmt.Sprintf("INSERT INTO timetable (line_id, bound, naptan_id, weekday, departure_time) VALUES %s",
		strings.Join(values, ",")))
	tx.MustExec(
		fmt.Sprintf("INSERT INTO timetable (line_id, bound, naptan_id, weekday, departure_time) VALUES %s",
			strings.Join(values, ",")))
	err := tx.Commit()
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Error(err.Error())
	}
	logger.GetLogger().Info("Replaced timetable for line %v (%v), stop %v", line, bound, origin)
}
func getRoutePaths(linesToPlot map[string]map[string]map[string]string) map[string][]LatLng {
	routePaths := make(map[string][]LatLng)
	bound := "outbound"

	for line, lineDetails := range linesToPlot {
		fromStop := lineDetails[bound]["fromStop"]
		toStop := lineDetails[bound]["toStop"]
		intermediateStops := selectdb.SelectIntermediateStops(line, bound, fromStop, toStop)

		path := make([]LatLng, len(intermediateStops))

		intermediateStopDetails := selectdb.SelectMultipleStops(intermediateStops)
		for i, stop := range intermediateStops {
			lat, err := strconv.ParseFloat(intermediateStopDetails[stop]["latitude"], 64)
			if err != nil {
				logger.GetLogger().Panic(err)
			}

			lng, err := strconv.ParseFloat(intermediateStopDetails[stop]["longitude"], 64)
			if err != nil {
				logger.GetLogger().Panic(err)
			}

			path[i] = LatLng{
				Lat: lat,
				Lng: lng,
			}
		}
		routePaths[line] = path
	}
	return routePaths
}
/* Returns timetable for specified line, bound and stop in the form of a map
   map (weekday => []("hour" => hour
                    "minute" => minute)) */
func SelectTimetable(line, bound, naptanId string) map[time.Weekday]map[LocalTime]bool {
	var timetable []TimetableEntry

	dbconn.AcquireLock()
	err := db.Select(&timetable,
		"SELECT weekday, departure_time FROM timetable WHERE line_id=$1 AND bound=$2 AND naptan_id=$3 ORDER BY weekday,departure_time",
		line, bound, naptanId)
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Panic(err)
	}

	result := make(map[time.Weekday]map[LocalTime]bool)
	for _, entry := range timetable {
		weekday := time.Weekday(entry.Weekday)
		weekdayMap, exists := result[weekday]
		if !exists {
			weekdayMap = make(map[LocalTime]bool, 0)
			result[weekday] = weekdayMap
		}
		weekdayMap[LocalTime{Hour: entry.DepartureTime.Hour(), Minute: entry.DepartureTime.Minute()}] = true
	}
	logger.GetLogger().Info("Get timetable for line %v (%v), stop %v returned: %v", line, bound, naptanId, result)
	return result
}
/* To handle the situation when stops logged from Tfl are not consecutive,
   i.e. fromStop and toStop are > 1 stop apart
   Applies interpolation to estimate neighbouring journey times, and
   updates the values for insertion into database

   @See interpolateTimes for implementation details */
func interpolateStops(line, bound, vehicleId, fromStop, toStop, oldTimeStamp, newTimeStamp, oldTflTimeStamp, newTflTimeStamp string,
	values, skippedStopsValues, rawArrivalTimes *[]string, cachedEntry map[string]string) {

	// Discard if interpolating too many stops (>= 10 stops)
	interpolationThreshold := 10

	// Find intermediate stops
	intermediateStops := selectdb.SelectIntermediateStops(line, bound, fromStop, toStop)
	intermediateStopsLength := len(intermediateStops)
	if intermediateStopsLength <= 2 {
		logger.GetLogger().Panicf("Expected at least one intermediate stop for line %v (%v), between %v and %v. But got none.",
			line, bound, fromStop, toStop)
	}

	if intermediateStopsLength >= interpolationThreshold {
		logger.GetLogger().Error("Bus %v of line %v (%v) skipped too many stops (%v stops), between %v and %v. Aborting interpolation.",
			vehicleId, line, bound, intermediateStopsLength, fromStop, toStop)
		return
	}

	/* Find out interpolated times */
	interpolatedTimes, err := interpolateTimes(line, bound, vehicleId, intermediateStops, oldTimeStamp, newTimeStamp, cachedEntry, rawArrivalTimes)

	if err != nil {
		logger.GetLogger().Error(err.Error())
		return
	}

	for i := 0; i < intermediateStopsLength-1; i++ {
		if interpolatedTimes[i].IsZero() || interpolatedTimes[i+1].IsZero() {
			// Check that record is not invalidated
			continue
		}
		timeDiff := int(interpolatedTimes[i+1].Sub(interpolatedTimes[i]).Seconds())

		*values = append(*values, fmt.Sprintf("('%v', '%v', '%v', '%v', TIMESTAMP '%v', TIMESTAMP '%v', "+
			"'%v', TIMESTAMP '%v', TIMESTAMP '%v', INTERVAL '%v seconds', TRUE)",
			line, bound, vehicleId,
			intermediateStops[i], timeToPsqlTimeStamp(interpolatedTimes[i]), toPsqlTimeStamp(oldTflTimeStamp),
			intermediateStops[i+1], timeToPsqlTimeStamp(interpolatedTimes[i+1]), toPsqlTimeStamp(newTflTimeStamp),
			timeDiff))
	}

	// Strip end points away from intermediate points before inserting into skipped_stops_history
	intermediateStopsEndPointsExclusive := intermediateStops[1 : intermediateStopsLength-1]
	for i, v := range intermediateStopsEndPointsExclusive {
		intermediateStopsEndPointsExclusive[i] = "'" + v + "'"
	}

	*skippedStopsValues = append(*skippedStopsValues,
		fmt.Sprintf("('%v', '%v', '%v', '%v', TIMESTAMP '%v', TIMESTAMP '%v', '%v', TIMESTAMP '%v', TIMESTAMP '%v', ARRAY[%v], %v)",
			line, bound, vehicleId,
			intermediateStops[0], timeToPsqlTimeStamp(interpolatedTimes[0]), toPsqlTimeStamp(oldTflTimeStamp),
			intermediateStops[intermediateStopsLength-1], timeToPsqlTimeStamp(interpolatedTimes[intermediateStopsLength-1]), toPsqlTimeStamp(newTflTimeStamp),
			strings.Join(intermediateStopsEndPointsExclusive, ","), len(intermediateStopsEndPointsExclusive)))
}
func parseLineStopPointsFromBody(body []byte) ([]string, map[string]map[string]string) {

	type StopPoint struct {
		Id        string  `json:"id"`
		IcsId     string  `json:"icsId"`
		StationId string  `json:"stationId"`
		Name      string  `json:"name"`
		Latitude  float64 `json:"lat"`
		Longitude float64 `json:"lon"`
	}
	type StopPointSequences struct {
		StopPt []StopPoint `json:"stopPoint"`
	}
	type Line struct {
		IsOutboundOnly bool                 `json:"isOutboundOnly"`
		LineId         string               `json:"lineId"`
		Direction      string               `json:"direction"`
		StopSequence   []StopPointSequences `json:"stopPointSequences"`
	}

	var line Line
	err := json.Unmarshal(body, &line)
	if err != nil {
		logger.GetLogger().Panicf("Tfl failed to give proper body: %v", string(body))
	}

	if line.IsOutboundOnly && line.Direction == "inbound" {
		logger.GetLogger().Info("Line %v is outbound only", line.LineId)
		return nil, nil
	}

	if len(line.StopSequence) == 0 {
		logger.GetLogger().Error("Failed to parse stop sequence for line %v (%v)", line.LineId, line.Direction)
		return nil, nil
	}

	stopPoints := line.StopSequence[0].StopPt

	result := make([]string, len(stopPoints))
	/* Get also stop info including name, icscode, lat, lng */
	stopInfos := make(map[string]map[string]string)
	for i, stopPoint := range stopPoints {
		result[i] = stopPoint.Id
		stopInfos[stopPoint.Id] = map[string]string{
			"name":          stopPoint.Name,
			"icscode":       stopPoint.IcsId,
			"latitude":      strconv.FormatFloat(stopPoint.Latitude, 'f', -1, 64),
			"longitude":     strconv.FormatFloat(stopPoint.Longitude, 'f', -1, 64),
			"stationNaptan": stopPoint.StationId,
		}
	}
	logger.GetLogger().Info("Parsed line %v (%v) for stops: %v", line.LineId, line.Direction, result)

	return result, stopInfos
}
func getStopInfos(lineStopPointsByRoute map[string]map[string][]string) map[string]map[string][]TflStopPoint {
	/* Get slice of all stops */
	stopsMap := make(map[string]bool)
	for _, lineDetails := range lineStopPointsByRoute {
		for _, orderedStopPoints := range lineDetails {
			for _, stop := range orderedStopPoints {
				stopsMap[stop] = true
			}
		}
	}
	stops := make([]string, len(stopsMap))
	counter := 0
	for stop, _ := range stopsMap {
		stops[counter] = stop
		counter++
	}

	stopInfos := selectdb.SelectMultipleStops(stops)

	result := make(map[string]map[string][]TflStopPoint)
	for line, lineDetails := range lineStopPointsByRoute {
		for bound, orderedStopPoints := range lineDetails {
			for _, naptanId := range orderedStopPoints {
				_, exists := result[line]
				if !exists {
					result[line] = make(map[string][]TflStopPoint)
				}
				_, exists = result[line][bound]
				if !exists {
					result[line][bound] = make([]TflStopPoint, 0)
				}

				stopInfo := stopInfos[naptanId]
				name := stopInfo["name"]
				latitude, err := strconv.ParseFloat(stopInfo["latitude"], 64)
				if err != nil {
					logger.GetLogger().Panic(err)
				}
				longitude, err := strconv.ParseFloat(stopInfo["longitude"], 64)
				if err != nil {
					logger.GetLogger().Panic(err)
				}

				result[line][bound] = append(result[line][bound],
					TflStopPoint{
						Name:      name,
						NaptanId:  naptanId,
						Latitude:  latitude,
						Longitude: longitude,
					})
			}
		}
	}
	return result
}
/* Returns the tfl predictions at the times specified for a given line, bound, origin and destination
   map time.Time => time.Duration */
func SelectTflPredictions(line, bound, fromStop, toStop string, times []time.Time) map[time.Time]time.Duration {
	tflPredictions := make([]int64, len(times))

	dbconn.AcquireLock()
	tx, err := db.Begin()
	for i, targetTime := range times {

		upperBound := targetTime.Add(10 * time.Minute)
		lowerBound := targetTime.Add(-10 * time.Minute)

		logger.GetLogger().Info(fmt.Sprintf("SELECT EXTRACT(epoch FROM predicted_time) FROM ("+
			"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
			"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time>='%v' AND departure_time <= '%v' ORDER BY departure_time LIMIT 1) "+
			"UNION ALL "+
			"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
			"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time<'%v' AND departure_time >= '%v' ORDER BY departure_time DESC LIMIT 1) "+
			") AS result ORDER BY abs(EXTRACT(epoch FROM departure_time-'%v')) LIMIT 1",
			line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(upperBound),
			line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(lowerBound),
			toPsqlTimeStamp(targetTime)))

		err = tx.QueryRow(
			fmt.Sprintf("SELECT EXTRACT(epoch FROM predicted_time) FROM ("+
				"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
				"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time>='%v' AND departure_time <= '%v' ORDER BY departure_time LIMIT 1) "+
				"UNION ALL "+
				"(SELECT predicted_time, departure_time FROM tfl_predictions_history "+
				"WHERE line='%v' AND bound='%v' AND from_id='%v' AND to_id='%v' AND departure_time<'%v' AND departure_time >= '%v' ORDER BY departure_time DESC LIMIT 1) "+
				") AS result ORDER BY abs(EXTRACT(epoch FROM departure_time-'%v')) LIMIT 1",
				line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(upperBound),
				line, bound, fromStop, toStop, toPsqlTimeStamp(targetTime), toPsqlTimeStamp(lowerBound),
				toPsqlTimeStamp(targetTime))).Scan(&tflPredictions[i])
		if err != nil {
			logger.GetLogger().Error(err.Error())
			tflPredictions[i] = 0
		}
	}
	err = tx.Commit()
	dbconn.ReleaseLock()

	if err != nil {
		logger.GetLogger().Panic(err)
	}

	result := make(map[time.Time]time.Duration)
	for i, t := range times {
		/* Skip if no recent prediction found (within 20 minute window) */
		tflPrediction := tflPredictions[i]
		if tflPrediction == 0 {
			continue
		}
		result[t] = time.Duration(tflPredictions[i]) * time.Second
	}
	return result
}
func parseCumulativeJourneyTimesFromBody(line, bound string, body []byte) []map[string]int {
	type Interval struct {
		StopId        string  `json:"stopId"`
		TimeToArrival float32 `json:"timeToArrival"`
	}

	type StationInterval struct {
		Intervals []Interval `json:"intervals"`
	}

	type Route struct {
		StationIntervals []StationInterval `json:"stationIntervals"`
	}

	type Times struct {
		Routes []Route `json:"routes"`
	}

	type Timetable struct {
		Timetable Times `json:"timetable"`
	}

	var timetable Timetable
	err := json.Unmarshal(body, &timetable)

	/* Sometimes Tfl returns "Technical error".
	   In such case we skip and return gracefully */
	if err != nil {
		logger.GetLogger().Error("Tfl failed to give proper arrivals response: %v", string(body))
		return make([]map[string]int, 0)
	}

	routes := timetable.Timetable.Routes
	if len(routes) == 0 {
		logger.GetLogger().Error("Failed to parse routes in timetable for line %v (%v)", line, bound)
		return make([]map[string]int, 0)
	}
	stationIntervals := routes[0].StationIntervals
	if len(stationIntervals) == 0 {
		logger.GetLogger().Error("Failed to parse station intervals in timetable for line %v (%v)", line, bound)
		return make([]map[string]int, 0)
	}

	intervals := stationIntervals[0].Intervals

	var result = make([]map[string]int, len(intervals))
	for i, interval := range intervals {
		m := make(map[string]int)
		m[interval.StopId] = int(interval.TimeToArrival)
		result[i] = m
	}
	logger.GetLogger().Info("Parsed cumulative journey times for line %v (%v): %v", line, bound, result)
	return result
}
/* Returns current, historic, and real time durations */
func getRouteDuration(line, bound string, orderedStopPoints []string,
	currentEstimates, currentHistoricEstimates, currentRealTimeEstimates map[string]map[string]time.Duration,
	delayCounts map[string]map[string]int) (time.Duration, time.Duration, time.Duration, error) {
	duration := time.Duration(0)
	historicDuration := time.Duration(0)
	realTimeDuration := time.Duration(0)

	for i := 0; i < len(orderedStopPoints)-1; i++ {
		fromStop := orderedStopPoints[i]
		toStop := orderedStopPoints[i+1]

		historicEstimate := time.Duration(0)
		realTimeEstimate := time.Duration(0)
		_, exists := currentHistoricEstimates[fromStop]
		if !exists {
			logger.GetLogger().Error("Lack current historic estimates for line %v (%v) fromStop: %v", line, bound, fromStop)
		} else {
			historicEstimate, exists = currentHistoricEstimates[fromStop][toStop]
			if !exists {
				logger.GetLogger().Error("Lack current historic estimates for line %v (%v) fromStop: %v, toStop: %v", line, bound, fromStop, toStop)
			}
		}

		_, exists = currentRealTimeEstimates[fromStop]
		if !exists {
			logger.GetLogger().Error("Lack current real time estimates for line %v (%v) fromStop: %v", line, bound, fromStop)
		} else {
			realTimeEstimate, exists = currentRealTimeEstimates[fromStop][toStop]
			if !exists {
				logger.GetLogger().Error("Lack current real time estimates for line %v (%v) fromStop: %v, toStop: %v", line, bound, fromStop, toStop)
			}
		}

		if historicEstimate != 0 {
			historicDuration += historicEstimate
		} else {
			historicDuration += realTimeEstimate
		}

		if realTimeEstimate != 0 {
			realTimeDuration += realTimeEstimate
		} else {
			realTimeDuration += historicEstimate
		}

		/* Use delay if trustworthy */
		if delayCounts[fromStop][toStop] >= 3 {
			duration += realTimeEstimate
		} else {
			duration += historicEstimate
		}
	}
	return duration, historicDuration, realTimeDuration, nil
}
/* Writes to .html file for resulting bus route map */
func writeBusMap(wr io.Writer, busMap BusMap, templatePath string) {
	t, err := template.New(path.Base(templatePath)).ParseFiles(templatePath)
	if err != nil {
		logger.GetLogger().Panic(err)
	}

	err = t.Execute(wr, busMap)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
}
/* Returns map (vehicleId => (
	   "direction" => bound, "naptanId" => naptanId,
	   "timeStamp" => timeStamp, "expectedArrival" => expectedArrival))
for each bus tracked */
func parseArrivalsFromBody(body []byte) map[string]map[string]string {
	type Arrivals struct {
		VehicleId       string `json:"vehicleId"`
		Direction       string `json:"direction"`
		NaptanId        string `json:"naptanId"`
		TimeStamp       string `json:"timeStamp"`
		TimeToStation   int    `json:"timeToStation"`
		ExpectedArrival string `json:"expectedArrival"`
	}

	var arrivals []Arrivals

	err := json.Unmarshal(body, &arrivals)

	/* Sometimes Tfl returns "Technical error".
	   In such case we skip and return gracefully */
	if err != nil {
		logger.GetLogger().Error("Tfl failed to give proper arrivals response: %v", string(body))
		return make(map[string]map[string]string)
	}

	result := make(map[string]map[string]string)
	for _, bus := range arrivals {
		bundle, bundleExists := result[bus.VehicleId]
		if !bundleExists {
			bundle = make(map[string]string)
		}

		/* Tfl api seems to return all future stops for a vehicle.
		   Ensures entry with lowest timeToStation is recorded in the map, thus pointing to the next stop
		   CATCH: Make sure timeToStation is not negative, i.e. a prediction persists even though it is
		          now a past event */
		timeToStation, recordAlreadyExists := bundle["timeToStation"]
		if recordAlreadyExists {
			recordedTimeToStation, _ := strconv.Atoi(timeToStation)
			if recordedTimeToStation < bus.TimeToStation {
				insertExpectedArrivalToBundle(bundle, bus.NaptanId, bus.ExpectedArrival)
				result[bus.VehicleId] = bundle
				continue
			}
		}
		bundle["direction"] = bus.Direction
		bundle["naptanId"] = bus.NaptanId
		bundle["timeStamp"] = bus.TimeStamp
		bundle["timeToStation"] = strconv.Itoa(bus.TimeToStation)
		bundle["expectedArrival"] = bus.ExpectedArrival
		insertExpectedArrivalToBundle(bundle, bus.NaptanId, bus.ExpectedArrival)
		result[bus.VehicleId] = bundle
	}

	logger.GetLogger().Info("Parsed arrivals from tfl: %v", result)
	return result
}
/* Plots the journey times throughout a day between a pair of stops */
func JourneyTimePlot(adjacentStops map[string]string, dates []time.Time,
	interval time.Duration, wr io.Writer, templatePath string) {
	var charts []JourneyTimeChart

	for fromStop, toStop := range adjacentStops {
		logger.GetLogger().Info("Handling journey: %v -> %v", fromStop, toStop)
		series := make([]JourneyTimeSeries, 0)

		var partitionedTimes []Partition
		for _, date := range dates {
			// Partition between startTime (start of day) and endTime (end of day)
			startTime := date.Truncate(24 * time.Hour)
			endTime := date.Truncate(24 * time.Hour).Add(24 * time.Hour)
			partitionedTimes = getPartitionedTimes(startTime, endTime, interval)

			// Get average journey times corresponding to partitions, and their 95% confidence intervals
			partitionedAvgJourneyTimes, stdDevJourneyTimes, invalidIndices := getAverageJourneyTimes(fromStop, toStop, partitionedTimes)
			lowerCI, upperCI, err := statistics.ConfidenceInterval(partitionedAvgJourneyTimes, stdDevJourneyTimes)

			if err != nil {
				logger.GetLogger().Panic(err)
			}

			// Insert into series the journey time of each partition to plot chart
			series = append(series, JourneyTimeSeries{date, partitionedAvgJourneyTimes, lowerCI, upperCI, invalidIndices,
				randomColour()})
		}

		// Generate chart - Write to *.html
		xlabs := make([]string, len(partitionedTimes))
		for i := 0; i < len(partitionedTimes); i++ {
			start := partitionedTimes[i].start
			// e.g. label x axis with time - e.g. 3:00 PM
			xlabs[i] = start.Format(time.Kitchen)
		}

		// Title of chart
		fromStopName := selectdb.SelectStop(fromStop)
		toStopName := selectdb.SelectStop(toStop)
		title := fmt.Sprintf("Journey times between %v and %v", fromStopName, toStopName)

		chart := JourneyTimeChart{
			title,
			fromStop,
			toStop,
			xlabs,
			series,
		}

		charts = append(charts, chart)
	}
	writeJourneyTimeCharts(wr, charts, templatePath)
}
func listFiles(dir string) []string {
	d, err := os.Open(dir)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	defer d.Close()

	names, err := d.Readdirnames(-1)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	return names
}
/* Returns map (vehicleId => (
	   "direction" => bound, "naptanId" => naptanId,
	   "timeStamp" => timeStamp, "expectedArrival" => expectedArrival))
for each bus tracked */
func parseStreamArrivalsFromResponse(response *http.Response) <-chan map[string]map[string]string {
	out := make(chan map[string]map[string]string)

	go func() {
		var predictionArray []json.Number
		dec := json.NewDecoder(response.Body)

		for dec.More() {
			err := dec.Decode(&predictionArray)
			if err != nil {
				logger.GetLogger().Panic(err)
			}

			if len(predictionArray) == 0 {
				logger.GetLogger().Panic("Unexpected empty prediction array")
			}

			responseType, _ := predictionArray[0].Int64()
			switch responseType {
			/* Response type 1 corresponds to prediction array.
			   Parse prediction array */
			case 1:
				if len(predictionArray) != 6 {
					logger.GetLogger().Info("Expected prediction array to have length 5, but got %v. Prediction array: %v",
						len(predictionArray), predictionArray)
				}

				naptanId := predictionArray[1].String()
				visitNumber := predictionArray[2].String()
				directionId, _ := predictionArray[3].Int64()
				vehicleId := predictionArray[4].String()
				expectedArrivalUnixEpoch, _ := predictionArray[5].Int64()

				result := make(map[string]map[string]string)
				bundle := make(map[string]string)
				bundle["naptanId"] = naptanId
				bundle["visitNumber"] = visitNumber
				bundle["direction"] = directionIdToBound(directionId, predictionArray)
				bundle["timeStamp"] = toTflTimeStamp(time.Now().In(timezone.Get()))
				bundle["expectedArrival"] = toTflTimeStamp(time.Unix(0, expectedArrivalUnixEpoch))
				result[vehicleId] = bundle

				out <- result
			default:
				logger.GetLogger().Error("Encountered non-prediction array: %v", predictionArray)
				continue
			}
		}
	}()
	return out
}
func main() {
	lines := selectdb.SelectLines()
	ch := make(chan string)
	for _, line_id := range lines {
		go func(line string) {
			updatedb.UpdateAdjacentStops(line)
			ch <- line
		}(line_id)
	}
	for i := 0; i < len(lines); i++ {
		logger.GetLogger().Info("Done for line %v", <-ch)
	}
	logger.GetLogger().Info("Adjacent stops for all lines updated")
}
func parseTflEstimateFromBody(line string, body []byte) time.Duration {

	type TflLineIdentifier struct {
		Id    string `json:"id"`
		Bound string `json:"bound"`
	}

	type TflRouteOption struct {
		Directions     []string          `json:"directions"`
		LineIdentifier TflLineIdentifier `json:"lineIdentifier"`
	}

	type TflLeg struct {
		Duration     int              `json:"duration"`
		RouteOptions []TflRouteOption `json:"routeOptions"`
	}

	type TflJourney struct {
		TflDuration int      `json:"duration"`
		Legs        []TflLeg `json:"legs"`
	}

	type TflDirections struct {
		Journeys []TflJourney `json:"journeys"`
	}

	var directions TflDirections
	err := json.Unmarshal(body, &directions)
	if err != nil {
		logger.GetLogger().Panicf("Tfl failed to give proper arrivals response: %v", string(body))
	}

	journeys := directions.Journeys
	if len(journeys) == 0 {
		logger.GetLogger().Error("Lack of journeys for route: %v", line)
		return 0
	}
	for _, journey := range journeys {
		for _, leg := range journey.Legs {
			for _, routeOption := range leg.RouteOptions {
				if routeOption.LineIdentifier.Id != line {
					continue
				}
			}
		}
		return time.Duration(journey.TflDuration) * time.Minute
	}
	return 0
}
/* Inserts credentials into site url request */
func GetResponseFromTfl(site string) *http.Response {
	site = fmt.Sprintf("%v&app_id=%v&app_key=%v", site,
		credentials.GetAppId(), credentials.GetAppKey())

	queryLimitChannel <- 0
	logger.GetLogger().Info("Attempting to access tfl site: %s", site)

	response, err := http.Get(site)
	if err != nil {
		logger.GetLogger().Panic(err)
	}

	logger.GetLogger().Info("Response from tfl status code: %v", response.StatusCode)
	return response
}
func getGoogleMapCredentials() GoogleMapCredentials {
	googleMapCredentialsFile, err := os.Open(googleMapCredentialsPath)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	defer googleMapCredentialsFile.Close()

	var credentials GoogleMapCredentials
	err = json.NewDecoder(googleMapCredentialsFile).Decode(&credentials)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	logger.GetLogger().Info("Go server credentials decoded - key: %v", credentials.Key)
	return credentials
}
func getTflCredentials() TflCredentials {
	tflCredentialsFile, err := os.Open(tflCredentialsPath)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	defer tflCredentialsFile.Close()

	var credentials TflCredentials
	err = json.NewDecoder(tflCredentialsFile).Decode(&credentials)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	logger.GetLogger().Info("Tfl credentials decoded - app_id: %v, app_key: %v", credentials.App_id, credentials.App_key)
	return credentials
}
func getTflStreamCredentials() TflStreamCredentials {
	tflStreamCredentialsFile, err := os.Open(tflStreamCredentialsPath)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	defer tflStreamCredentialsFile.Close()

	var credentials TflStreamCredentials
	err = json.NewDecoder(tflStreamCredentialsFile).Decode(&credentials)
	if err != nil {
		logger.GetLogger().Panic(err)
	}
	logger.GetLogger().Info("Tfl stream credentials decoded - username: %v, password: %v", credentials.Username, credentials.Password)
	return credentials
}