// Load all the messages, and sort into a single time-ordered stream.
func file2msgs(filename string) []adsb.Msg {
file, err := os.Open(filename)
if err != nil {
log.Fatal(err)
}
defer file.Close()
msgs := []adsb.Msg{}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
msg := adsb.Msg{}
if err := msg.FromSBS1(scanner.Text()); err != nil {
log.Fatal("Bad parse '%s'\n%v\n", scanner.Text(), err)
}
// We drop the useless "111" fields, so this doesn't work. Maybe add them in, for ToSBS1 ?
//if msg.ToSBS1() != scanner.Text() {
// log.Fatalf("parse/print fail\nread: %s\ngen : %s\n", scanner.Text(), msg.ToSBS1())
//}
msgs = append(msgs, msg)
}
return msgs
}
func main() {
Log.Printf("reading file '%s' (dumpPos=%v)", fFilename, fDumpPos)
h := histogram.Histogram{
ValMin: 0,
ValMax: 80,
NumBuckets: 80,
}
_ = h
if osFile, err := os.Open(fFilename); err != nil {
Log.Fatal(err)
} else {
scanner := bufio.NewScanner(osFile) // os.File implements io.Reader
for scanner.Scan() {
m := adsb.Msg{}
text := scanner.Text()
if err := m.FromSBS1(text); err != nil {
Log.Fatal(err)
break
}
if fDumpPos {
if m.HasPosition() {
fmt.Printf("\"%.5f,%.5f\"\n", m.Position.Lat, m.Position.Long)
}
} else {
Log.Print(m)
}
}
if err := scanner.Err(); err != nil {
Log.Fatal(err)
}
}
}
// MaybeAdd looks at a new message, and updates the buffer as appropriate.
func (mb *MsgBuffer) Add(m *adsb.Msg) {
mb.ageOutQuietSenders()
if _, exists := mb.Senders[m.Icao24]; exists == false {
// We've not seen this sender before. If we have position data,
// start the whitelisting thing. We only Whitelist senders who
// will eventually send useful info (e.g. position), so wait until
// we see that.
if m.HasPosition() {
mb.Senders[m.Icao24] = &ADSBSender{LastSeen: time.Now().UTC()}
}
} else {
mb.Senders[m.Icao24].updateFromMsg(m) // Pluck out anything interesting
if composite := mb.Senders[m.Icao24].maybeCreateComposite(m); composite != nil {
// We have a message to store !!
mb.Messages = append(mb.Messages, composite)
}
}
// We use the timestamp in the message to decide when to flush,
// rather than the time at which we received the message; this is to
// deliver a better end-to-end QoS for message delivery.
// But stale messages can arrive, with timestamps from the past;
// they would always trigger a flush, and flushing every message
// slows things down (so we never ever catch up again :().
// So we also enforce a minimum interval between flushes.
if len(mb.Messages) > 0 {
t := mb.Messages[0].GeneratedTimestampUTC
if time.Since(t) >= mb.MaxMessageAge && time.Since(mb.lastFlush) >= mb.MinPublishInterval {
mb.flush()
}
}
}
// If this message has new position info, *and* we have good backfill, then craft a CompositeMsg.
// Note, we don't wait for squawk info.
func (s *ADSBSender) maybeCreateComposite(m *adsb.Msg) *adsb.CompositeMsg {
if !m.HasPosition() {
return nil
}
//if s.LastGroundSpeed == 0 || s.LastTrack == 0 || s.LastCallsign == "" { return nil }
cm := adsb.CompositeMsg{Msg: *m} // Clone the input into the embedded struct
// Overwrite with cached info (from previous packets), if we don't have it in this packet
if cm.GroundSpeed == 0 {
cm.GroundSpeed = s.LastGroundSpeed
}
if cm.VerticalRate == 0 {
cm.VerticalRate = s.LastVerticalSpeed
}
if cm.Track == 0 {
cm.Track = s.LastTrack
}
if cm.Callsign == "" {
cm.Callsign = s.LastCallsign
}
if cm.Squawk == "" {
cm.Squawk = s.LastSquawk
}
return &cm
}
func msgs(sbs string) (ret []adsb.Msg) {
scanner := bufio.NewScanner(strings.NewReader(sbs))
for scanner.Scan() {
m := adsb.Msg{}
text := scanner.Text()
if err := m.FromSBS1(text); err != nil {
panic(err)
}
m.GeneratedTimestampUTC = time.Now() // Fake this out
ret = append(ret, m)
}
return
}
// Parse up the SBS strings, and then pretend we've fleshed them out with data into CompositeMsgs
func msgs(sbsText string) (ret []*adsb.CompositeMsg) {
scanner := bufio.NewScanner(strings.NewReader(sbsText))
for scanner.Scan() {
if text := scanner.Text(); text != "" {
m := adsb.Msg{}
if err := m.FromSBS1(text); err != nil {
panic(err)
}
cm := adsb.CompositeMsg{Msg: m}
ret = append(ret, &cm)
}
}
return
}
func TestAgeOutQuietSenders(t *testing.T) {
mb := NewMsgBuffer()
messages := msgs(maybeAddSBS)
for _, msg := range messages {
mb.Add(&msg)
}
unrelatedMsg := adsb.Msg{}
if err := unrelatedMsg.FromSBS1(unrelatedSBS); err != nil {
panic(err)
}
// Pluck out the (only) sender ID, reset its clock into the past
var id adsb.IcaoId
for k, _ := range mb.Senders {
id = k
}
// Rig time - just before the age out window
offset := -1*mb.MaxQuietTime - 5
mb.Senders[id].LastSeen = mb.Senders[id].LastSeen.Add(offset)
mb.Add(&unrelatedMsg) // Send a message, to trigger ageout
if len(mb.Senders) != 1 {
t.Errorf("aged out too soon ?")
}
// Rig time - just after the age out window. And reset the sweep time.
mb.Senders[id].LastSeen = mb.Senders[id].LastSeen.Add(time.Duration(-10) * time.Second)
mb.lastAgeOut = mb.lastAgeOut.Add(time.Second * time.Duration(-5))
mb.Add(&unrelatedMsg) // Send a message, to trigger ageout
if len(mb.Senders) != 0 {
t.Errorf("aged out, but still present")
}
_ = fmt.Sprintf("%s", mb)
}
func generateData() {
fmt.Printf("(launching mock dump0190 on localhost:%d; mlat=%v)\n", port, mlat)
ln, _ := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
outerLoop:
for {
conn, _ := ln.Accept()
fmt.Printf("(connection started)\n")
m := adsb.Msg{
Icao24: adsb.IcaoId("A81BD0"),
Callsign: "ABW123",
Type: "MSG",
Altitude: 12345,
GroundSpeed: 300,
Track: 315,
VerticalRate: 64,
Position: geo.Latlong{36.0, -122.0},
GeneratedTimestampUTC: time.Now().UTC(),
LoggedTimestampUTC: time.Now().UTC(),
}
// We need to prime the pump, and trick the msgbuffer
m.SubType = 3 // Get an entry in the sender table for our Icao, by proving we have pos data
conn.Write([]byte(fmt.Sprintf("%s\n", m.ToSBS1())))
m.SubType = 1 // Populate the sender table entry with a callsign (MSG,1 only)
conn.Write([]byte(fmt.Sprintf("%s\n", m.ToSBS1())))
m.SubType = 4 // Populate the sender table entry with velocity data (MSG,4 only)
conn.Write([]byte(fmt.Sprintf("%s\n", m.ToSBS1())))
m.SubType = 3 // All future messages are linear position updates (MSG,3 only)
for {
now := time.Now().UTC().Add(-1 * delay)
m.Position.Lat += 0.01
m.GeneratedTimestampUTC = now
m.LoggedTimestampUTC = now
if mlat {
m.Type = "MLAT"
}
if _, err := conn.Write([]byte(fmt.Sprintf("%s\n", m.ToSBS1()))); err != nil {
fmt.Printf("(connection ended)\n")
continue outerLoop
}
time.Sleep(time.Millisecond * 1000)
}
}
}
// Some subtype packets have data we don't get in the bulk of position packets (those of subtype:3),
// so just cache their interesting data and inject it into next position packet.
// http://woodair.net/SBS/Article/Barebones42_Socket_Data.htm
func (s *ADSBSender) updateFromMsg(m *adsb.Msg) {
s.LastSeen = time.Now().UTC()
// If the message had any of the optional fields, cache the value for later
if m.HasCallsign() {
if len(m.Callsign) > 0 {
s.LastCallsign = m.Callsign
} else {
s.LastCallsign = "_._._._." // Our nil value :/
}
}
if m.HasSquawk() {
s.LastSquawk = m.Squawk
}
if m.HasGroundSpeed() {
s.LastGroundSpeed = m.GroundSpeed
}
if m.HasTrack() {
s.LastTrack = m.Track
}
if m.HasVerticalRate() {
s.LastVerticalSpeed = m.VerticalRate
}
if m.Type == "MSG_foooo" {
if m.SubType == 1 {
// TODO: move this to m.hasCallsign()
// MSG,1 - the callsign/ident subtype - is sometimes blank. But we
// don't really want to confuse flights that have a purposefully
// blank callsign with those for yet we've yet to receive a MSG,1.
// So we use a magic string instead.
if m.Callsign == "" {
s.LastCallsign = "_._._._."
}
if m.Callsign != "" {
s.LastCallsign = m.Callsign
}
} else if m.SubType == 2 {
if m.HasGroundSpeed() {
s.LastGroundSpeed = m.GroundSpeed
}
if m.HasTrack() {
s.LastTrack = m.Track
}
} else if m.SubType == 4 {
if m.HasGroundSpeed() {
s.LastGroundSpeed = m.GroundSpeed
}
if m.HasVerticalRate() {
s.LastVerticalSpeed = m.VerticalRate
}
if m.HasTrack() {
s.LastTrack = m.Track
}
} else if m.SubType == 6 {
if m.Squawk != "" {
s.LastSquawk = m.Squawk
}
}
}
}
// readMsgFromSocket will pull basestation (and extended basestation)
// formatted messages from the socket, and send them down the channel.
// It will retry the connection on failure.
func readMsgFromSocket(wg *sync.WaitGroup, hostport string, msgChan chan<- *adsb.Msg) {
nTimeMismatches := 0
lastBackoff := time.Second
wg.Add(1)
outerLoop:
for {
if weAreDone() {
break
} // outer
conn, err := net.Dial("tcp", hostport)
if err != nil {
Log.Printf("connect '%s': err %s; trying again in %s ...", hostport, err, lastBackoff*2)
time.Sleep(lastBackoff)
if lastBackoff < time.Minute*5 {
lastBackoff *= 2
}
continue
}
lastBackoff = time.Second
Log.Printf("connected to '%s'", hostport)
// a net.Conn implements io.Reader
scanner := bufio.NewScanner(conn)
for scanner.Scan() { // This can block indefinitely ...
if weAreDone() {
break outerLoop
}
if err := scanner.Err(); err != nil {
Log.Printf("killing connection, scanner err: %v\n", err)
conn.Close()
break // inner
}
msg := adsb.Msg{}
text := scanner.Text()
if err := msg.FromSBS1(text); err != nil {
Log.Printf("killing connection, SBS input:%q, parse fail: %v", text, err)
break // inner
}
// If there is significant clock skew, we should bail. But, it seems
// that sometimes we pick up stale data from dump1090; so wait to see
// if it passes.
offset := time.Since(msg.GeneratedTimestampUTC)
if offset > time.Minute*30 || offset < time.Minute*-30 {
nTimeMismatches++
if nTimeMismatches < 100 {
continue // do not process this message
} else {
Log.Fatalf("100 bad msgs; set -timeloc ?\nNow = %s\nmsg = %s\n", time.Now(),
msg.GeneratedTimestampUTC)
}
}
// If the message is flagged as one we should mask, honor that
if msg.IsMasked() {
continue
}
msgChan <- &msg
}
}
wg.Done()
Log.Printf(" ---- readMsgFromSocket, clean shutdown\n")
}