func TestConstituents(t *testing.T) { gauges := make(map[string]meta.Gauge) { t.Log("Load installed gauges file") var list meta.GaugeList if err := meta.LoadList("../network/gauges.csv", &list); err != nil { t.Fatal(err) } for _, g := range list { gauges[g.Code] = g } } var constituents meta.ConstituentList if err := meta.LoadList("../network/constituents.csv", &constituents); err != nil { t.Fatal(err) } for i := 0; i < len(constituents); i++ { for j := i + 1; j < len(constituents); j++ { if constituents[i].Gauge == constituents[j].Gauge && constituents[i].Number == constituents[j].Number { t.Error("contituent duplication: " + constituents[i].Gauge + "/" + strconv.Itoa(constituents[i].Number)) } } } for _, c := range constituents { if _, ok := gauges[c.Gauge]; !ok { t.Error("unknown gauge: " + c.Gauge) } } }
func TestMonuments(t *testing.T) { var monuments meta.MonumentList t.Log("Load network monuments file") { if err := meta.LoadList("../network/monuments.csv", &monuments); err != nil { t.Fatal(err) } } for i := 0; i < len(monuments); i++ { for j := i + 1; j < len(monuments); j++ { if monuments[i].Mark == monuments[j].Mark { t.Errorf("monument duplication: " + monuments[i].Mark) } } } for _, m := range monuments { if m.GroundRelationship > 0.0 { t.Errorf("positive monuments ground relationship: %s [%g]", m.Mark, m.GroundRelationship) } switch m.Type { case "Shallow Rod / Braced Antenna Mount": case "Wyatt/Agnew Drilled-Braced": case "Pillar": case "Steel Mast": case "Unknown": default: t.Errorf("unknown monument type: %s [%s]", m.Mark, m.Type) } } }
func TestGauges(t *testing.T) { stas := make(map[string]meta.Station) { var list meta.StationList t.Log("Load stations file") if err := meta.LoadList("../network/stations.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { stas[s.Code] = s } } var gauges meta.GaugeList t.Log("Load installed gauges file") { if err := meta.LoadList("../network/gauges.csv", &gauges); err != nil { t.Fatal(err) } } for i := 0; i < len(gauges); i++ { for j := i + 1; j < len(gauges); j++ { if gauges[i].Code == gauges[j].Code { t.Errorf("gauge code duplication: " + gauges[i].Code) } if gauges[i].Number == gauges[j].Number { t.Errorf("gauge number duplication: " + gauges[i].Code) } } } for _, g := range gauges { if _, ok := stas[g.Code]; !ok { t.Error("unknown gauge station: " + g.Code) } } }
func TestSites(t *testing.T) { var sites meta.SiteList t.Log("Load installed sites file") { if err := meta.LoadList("../network/sites.csv", &sites); err != nil { t.Fatal(err) } } for i := 0; i < len(sites); i++ { for j := i + 1; j < len(sites); j++ { if sites[i].Station == sites[j].Station && sites[i].Location == sites[j].Location { t.Errorf("site duplication: " + sites[i].Station + "/" + sites[i].Location) } } } }
func TestNetworks(t *testing.T) { var networks meta.NetworkList t.Log("Load installed sensors file") { if err := meta.LoadList("../network/networks.csv", &networks); err != nil { t.Fatal(err) } } for i := 0; i < len(networks); i++ { for j := i + 1; j < len(networks); j++ { if networks[i].Code == networks[j].Code { t.Errorf("network duplication: " + networks[i].Code) } } } }
func TestStations(t *testing.T) { var stations meta.StationList t.Log("Load installed sensors file") { if err := meta.LoadList("../network/stations.csv", &stations); err != nil { t.Fatal(err) } } for i := 0; i < len(stations); i++ { for j := i + 1; j < len(stations); j++ { if stations[i].Code == stations[j].Code { t.Errorf("station duplication: " + stations[i].Code) } } } }
func TestMarks(t *testing.T) { var marks meta.MarkList t.Log("Load network marks file") { if err := meta.LoadList("../network/marks.csv", &marks); err != nil { t.Fatal(err) } } for i := 0; i < len(marks); i++ { for j := i + 1; j < len(marks); j++ { if marks[i].Code == marks[j].Code { t.Errorf("mark duplication: " + marks[i].Code) } } } }
func TestRecorders(t *testing.T) { var recorders meta.InstalledRecorderList t.Log("Load installed recorders file") { if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil { t.Fatal(err) } } t.Log("Check for recorder installation equipment overlaps") { installs := make(map[string]meta.InstalledRecorderList) for _, s := range recorders { _, ok := installs[s.Model] if ok { installs[s.Model] = append(installs[s.Model], s) } else { installs[s.Model] = meta.InstalledRecorderList{s} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("recorder %s/%s at %-5s has location %-2s overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for missing recorder stations") { var stations meta.StationList if err := meta.LoadList("../network/stations.csv", &stations); err != nil { t.Fatal(err) } keys := make(map[string]interface{}) for _, s := range stations { keys[s.Code] = true } for _, s := range recorders { if _, ok := keys[s.Station]; ok { continue } t.Errorf("unable to find recorder installed station %-5s", s.Station) } } var assets meta.AssetList t.Log("Load recorder assets file") { if err := meta.LoadList("../assets/recorders.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for recorder assets") { for _, r := range recorders { model := r.DataloggerModel if r.DataloggerModel != r.Model { model = strings.Join([]string{r.DataloggerModel, r.Model}, " ") } var found bool for _, a := range assets { if a.Model != model { continue } if a.Serial != r.Serial { continue } found = true } if !found { t.Errorf("unable to find recorders asset: %s [%s]", model, r.Serial) } } } }
// TestStationsProto creates a Protobuf file of Stations. func TestStationsProto(t *testing.T) { var networks meta.NetworkList if err := meta.LoadList("../network/networks.csv", &networks); err != nil { t.Error(err) } var net = make(map[string]*delta.Network) for _, v := range networks { n := delta.Network{ Code: v.Code, External: v.External, Description: v.Description, Restricted: v.Restricted, } net[v.Code] = &n } var stations meta.StationList if err := meta.LoadList("../network/stations.csv", &stations); err != nil { t.Error(err) } if len(stations) == 0 { t.Error("zero length stations list.") } var s delta.Stations s.Stations = make(map[string]*delta.Station) for _, v := range stations { pt := delta.Point{ Longitude: v.Longitude, Latitude: v.Latitude, Elevation: v.Elevation, Datum: v.Datum, } sp := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } st := delta.Station{ Code: v.Code, Name: v.Name, Network: net[v.Network], Point: &pt, Span: &sp, } s.Stations[st.Code] = &st } b, err := proto.Marshal(&s) if err != nil { t.Error(err) } t.Log(s) if err := os.MkdirAll(apiDir, 0777); err != nil { t.Error(err) } if err := ioutil.WriteFile(apiDir+"/stations.pb", b, 0644); err != nil { t.Error(err) } }
// TestMarksProto creates protobuf and JSON files of Marks. // These are pushed to S3 (by Travis) for use in api.geonet.org.nz // Three files are created: // marks.pb - fully hydrated protobuf will all GNSS Mark information. // marks.json - JSON version of marks.pb (for use in browsers). // marks.geojson - GeoJSON of Mark locations. func TestMarksProto(t *testing.T) { var networks meta.NetworkList if err := meta.LoadList("../network/networks.csv", &networks); err != nil { t.Error(err) } var net = make(map[string]*delta.Network) for _, v := range networks { n := delta.Network{ Code: v.Code, External: v.External, Description: v.Description, Restricted: v.Restricted, } net[v.Code] = &n } var marks meta.MarkList if err := meta.LoadList("../network/marks.csv", &marks); err != nil { t.Error(err) } if len(marks) == 0 { t.Error("zero length mark list.") } var m delta.Marks m.Marks = make(map[string]*delta.Mark) for _, v := range marks { pt := delta.Point{ Longitude: v.Longitude, Latitude: v.Latitude, Elevation: v.Elevation, Datum: v.Datum, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } mk := delta.Mark{ Code: v.Code, Name: v.Name, Network: net[v.Network], Point: &pt, Span: &s, } m.Marks[mk.Code] = &mk } var monuments meta.MonumentList if err := meta.LoadList("../network/monuments.csv", &monuments); err != nil { t.Error(err) } for _, v := range monuments { s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } mn := delta.Monument{ DomesNumber: v.DomesNumber, MarkType: v.MarkType, Type: v.Type, GroundRelationship: v.GroundRelationship, FoundationType: v.FoundationType, FoundationDepth: v.FoundationDepth, Bedrock: v.Bedrock, Geology: v.Geology, Span: &s, } if _, ok := m.Marks[v.Mark]; ok { m.Marks[v.Mark].Monument = &mn } } var antennas meta.InstalledAntennaList if err := meta.LoadList("../install/antennas.csv", &antennas); err != nil { t.Error(err) } for _, v := range antennas { e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, } o := delta.Offset{ Vertical: v.Vertical, North: v.North, East: v.East, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } a := delta.InstalledAntenna{ Equipment: &e, Offset: &o, Span: &s, Azimuth: v.Azimuth, } if _, ok := m.Marks[v.Mark]; ok { m.Marks[v.Mark].InstalledAntenna = append(m.Marks[v.Mark].InstalledAntenna, &a) } } var radomes meta.InstalledRadomeList if err := meta.LoadList("../install/radomes.csv", &radomes); err != nil { t.Error(err) } for _, v := range radomes { e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } r := delta.InstalledRadome{ Equipment: &e, Span: &s, } if _, ok := m.Marks[v.Mark]; ok { m.Marks[v.Mark].InstalledRadome = append(m.Marks[v.Mark].InstalledRadome, &r) } } var firmwares meta.FirmwareHistoryList if err := meta.LoadList("../install/firmware.csv", &firmwares); err != nil { t.Error(err) } var fw = make(map[delta.Equipment]delta.Receiver) for _, v := range firmwares { e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } f := delta.Firmware{ Version: v.Version, Notes: v.Notes, Span: &s, } rx := fw[e] rx.Equipment = &e rx.Firmware = append(rx.Firmware, &f) fw[e] = rx } var receivers meta.DeployedReceiverList if err := meta.LoadList("../install/receivers.csv", &receivers); err != nil { t.Error(err) } for _, v := range receivers { e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } rx, ok := fw[e] if !ok { t.Errorf("no firware for %v", e) } d := delta.DeployedReceiver{ Receiver: &rx, Span: &s, } if _, ok := m.Marks[v.Mark]; ok { m.Marks[v.Mark].DeployedReceiver = append(m.Marks[v.Mark].DeployedReceiver, &d) } } var sessions meta.SessionList if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil { t.Error(err) } for _, v := range sessions { s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } se := delta.Session{ Operator: v.Operator, Agency: v.Agency, Model: v.Model, SatelliteSystem: v.SatelliteSystem, Interval: v.Interval.Nanoseconds(), ElevationMask: v.ElevationMask, HeaderComment: v.HeaderComment, Span: &s, } if _, ok := m.Marks[v.Mark]; ok { m.Marks[v.Mark].Session = append(m.Marks[v.Mark].Session, &se) } } var metsensors meta.InstalledMetSensorList if err := meta.LoadList("../install/metsensors.csv", &metsensors); err != nil { t.Error(err) } for _, v := range metsensors { e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } p := delta.Point{ Longitude: v.Longitude, Latitude: v.Latitude, Elevation: v.Elevation, Datum: v.Datum, } ms := delta.InstalledMetSensor{ Equipment: &e, Span: &s, Point: &p, IMSComment: v.IMSComment, } if _, ok := m.Marks[v.Mark]; ok { m.Marks[v.Mark].InstalledMetSensor = append(m.Marks[v.Mark].InstalledMetSensor, &ms) } } // output files if err := os.MkdirAll(apiDir, 0777); err != nil { t.Error(err) } b, err := proto.Marshal(&m) if err != nil { t.Error(err) } if err := ioutil.WriteFile(apiDir+"/marks.pb", b, 0644); err != nil { t.Error(err) } b, err = json.Marshal(&m) if err != nil { t.Error(err) } if err := ioutil.WriteFile(apiDir+"/marks.json", b, 0644); err != nil { t.Error(err) } // GeoJSON files of site for each sensor type. // This is similar to the sensor type output in stations_test.go // There is no other sensor type lookup so handle that switching here. // There is fractionally more work as this is two different installed types // (antenna and metsensor). out := map[string]*bytes.Buffer{ "metsensor": &bytes.Buffer{}, "gpsantenna": &bytes.Buffer{}, } for k := range out { out[k].WriteString(`{"type": "FeatureCollection","features": [`) } for _, mark := range m.Marks { if mark.GetSpan() == nil || mark.GetPoint() == nil || mark.GetNetwork() == nil { continue } if mark.InstalledAntenna != nil && len(mark.InstalledAntenna) > 0 { for _, v := range mark.InstalledAntenna { writeAntennaProps(mark, v, out["gpsantenna"]) } } if mark.InstalledMetSensor != nil && len(mark.InstalledMetSensor) > 0 { for _, v := range mark.InstalledMetSensor { writeMetsensorProps(mark, v, out["metsensor"]) } } } for k := range out { out[k].WriteString(`]}`) fn := apiDir + "/" + k + ".geojson" if err := ioutil.WriteFile(fn, out[k].Bytes(), 0644); err != nil { t.Error(err) } } }
func TestSensors(t *testing.T) { var installed meta.InstalledSensorList t.Log("Load installed sensors file") { if err := meta.LoadList("../install/sensors.csv", &installed); err != nil { t.Fatal(err) } sort.Sort(installed) } t.Log("Check for missing sensors") { var sensors meta.AssetList if err := meta.LoadList("../assets/sensors.csv", &sensors); err != nil { t.Fatal(err) } sort.Sort(sensors) for _, i := range installed { n := sort.Search(len(sensors), func(j int) bool { return !sensors[j].Equipment.Less(i.Equipment) }) if n < 0 || i.Equipment.Less(sensors[n].Equipment) { t.Errorf("unable to find sensor: %s", i.String()) } } } t.Log("Check for sensor installation overlaps") { installs := make(map[string]meta.InstalledSensorList) for _, s := range installed { _, ok := installs[s.Model] if ok { installs[s.Model] = append(installs[s.Model], s) } else { installs[s.Model] = meta.InstalledSensorList{s} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("sensor %s/%s at %-5s has location %-2s overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } /* t.Log("Check for missing sensor stations") { var stations meta.StationList if err := meta.LoadList("../network/stations.csv", &stations); err != nil { t.Fatal(err) } sort.Sort(stations) for _, i := range installed { n := sort.Search(len(stations), func(j int) bool { return !(stations[j].Code < i.StationCode) }) if n < 0 || i.StationCode != stations[n].Code { t.Errorf("unable to find station: %s", i.StationCode) } else { if i.Start.Before(stations[n].Start) { t.Errorf("installed sensor before station has been opened: %s: %s (%s %s)", i.String(), i.Start.String(), stations[n].Code, stations[n].Start.String()) } if i.End.After(stations[n].End) { t.Errorf("installed sensor after station has been closed: %s: %s (%s %s)", i.String(), i.End.String(), stations[n].Code, stations[n].End.String()) } } } } t.Log("Check for missing sensor sites") { var sites meta.SiteList if err := meta.LoadList("../network/sites.csv", &sites); err != nil { t.Fatal(err) } sort.Sort(sites) for _, i := range installed { n := sort.Search(len(sites), func(j int) bool { if sites[j].StationCode > i.StationCode { return true } if sites[j].StationCode < i.StationCode { return false } return !(sites[j].LocationCode < i.LocationCode) }) if n < 0 || i.StationCode != sites[n].StationCode || i.LocationCode != sites[n].LocationCode { t.Errorf("unable to find site: %s/%s", i.StationCode, i.LocationCode) } else { if i.Start.Before(sites[n].Start) { t.Errorf("installed sensor before site has been opened: %s: %s (%s/%s %s)", i.String(), i.Start.String(), sites[n].StationCode, sites[n].LocationCode, sites[n].Start.String()) } if i.End.After(sites[n].End) { t.Errorf("installed sensor after site has been closed: %s: %s (%s/%s %s)", i.String(), i.End.String(), sites[n].StationCode, sites[n].LocationCode, sites[n].End.String()) } } } } */ for _, i := range installed { if i.Orientation.Azimuth < -360.0 || i.Orientation.Azimuth > 360.0 { t.Errorf("installed sensor has invalid orientation azimuth: %s [%g]", i.String(), i.Orientation.Azimuth) } if i.Orientation.Dip < -90.0 || i.Orientation.Dip > 90.0 { t.Errorf("installed sensor has invalid orientation dip: %s [%g]", i.String(), i.Orientation.Dip) } } var assets meta.AssetList t.Log("Load receiver sensors file") { if err := meta.LoadList("../assets/sensors.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for sensor assets") { for _, s := range installed { var found bool for _, a := range assets { if a.Model != s.Model { continue } if a.Serial != s.Serial { continue } found = true } if !found { t.Errorf("unable to find sensor asset: %s [%s]", s.Model, s.Serial) } } } }
func TestReceivers(t *testing.T) { var receivers meta.DeployedReceiverList t.Log("Load deployed receivers file") { if err := meta.LoadList("../install/receivers.csv", &receivers); err != nil { t.Fatal(err) } } t.Log("Check for particular receiver installation overlaps") { installs := make(map[string]meta.DeployedReceiverList) for _, s := range receivers { if _, ok := installs[s.Model]; !ok { installs[s.Model] = meta.DeployedReceiverList{} } installs[s.Model] = append(installs[s.Model], s) } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("receiver %s [%s] at %s has overlap with %s between times %s and %s", v[i].Model, v[i].Serial, v[i].Mark, v[j].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for receiver sites installation equipment overlaps") { installs := make(map[string]meta.DeployedReceiverList) for _, s := range receivers { if _, ok := installs[s.Mark]; !ok { installs[s.Mark] = meta.DeployedReceiverList{} } installs[s.Mark] = append(installs[s.Model], s) } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("receivers %s [%s] / %s [%s] at %s has overlap between %s and %s", v[i].Model, v[i].Serial, v[j].Model, v[j].Serial, v[i].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for missing receiver marks") { var marks meta.MarkList if err := meta.LoadList("../network/marks.csv", &marks); err != nil { t.Fatal(err) } keys := make(map[string]interface{}) for _, m := range marks { keys[m.Code] = true } for _, r := range receivers { if _, ok := keys[r.Mark]; ok { continue } t.Errorf("unable to find receiver mark %-5s", r.Mark) } } var assets meta.AssetList t.Log("Load receiver assets file") { if err := meta.LoadList("../assets/receivers.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for receiver assets") { for _, r := range receivers { var found bool for _, a := range assets { if a.Model != r.Model { continue } if a.Serial != r.Serial { continue } found = true } if !found { t.Errorf("unable to find receiver asset: %s [%s]", r.Model, r.Serial) } } } }
func TestConnections(t *testing.T) { var connections meta.ConnectionList t.Log("Load connections file") if err := meta.LoadList("../install/connections.csv", &connections); err != nil { t.Fatal(err) } for i := 0; i < len(connections); i++ { for j := i + 1; j < len(connections); j++ { if connections[i].Station != connections[j].Station { continue } if connections[i].Location != connections[j].Location { continue } if connections[i].Start.After(connections[j].End) { continue } if connections[i].End.Before(connections[j].Start) { continue } t.Errorf("connection overlap: " + strings.Join([]string{ connections[i].Station, connections[i].Location, connections[i].Start.String(), connections[i].End.String(), }, " ")) } } stas := make(map[string]meta.Station) { var list meta.StationList t.Log("Load stations file") if err := meta.LoadList("../network/stations.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { stas[s.Code] = s } } sites := make(map[string]map[string]meta.Site) { var list meta.SiteList t.Log("Load sites file") if err := meta.LoadList("../network/sites.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { if _, ok := sites[s.Station]; !ok { sites[s.Station] = make(map[string]meta.Site) } sites[s.Station][s.Location] = s } } for _, c := range connections { if _, ok := stas[c.Station]; !ok { t.Log("unknown connection station: " + c.Station) } else if s, ok := sites[c.Station]; !ok { t.Log("unknown connection station: " + c.Station) } else if _, ok := s[c.Location]; !ok { t.Log("unknown connection station/location: " + c.Station + "/" + c.Location) } if c.Start.After(c.End) { t.Log("connection span mismatch: " + strings.Join([]string{ c.Station, c.Location, c.Start.String(), "after", c.End.String(), }, " ")) } } places := make(map[string]string) { var list meta.DeployedDataloggerList t.Log("Load installed dataloggers file") if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil { t.Fatal(err) } for _, d := range list { if d.Role != "" { places[d.Place+"/"+d.Role] = d.Place } else { places[d.Place] = d.Place } } } for _, c := range connections { if c.Role != "" { if _, ok := places[c.Place+"/"+c.Role]; !ok { t.Log("warning: unknown datalogger place/role: " + c.Place + "/" + c.Role) } } else { if _, ok := places[c.Place]; !ok { t.Log("warning: unknown datalogger place: " + c.Place) } } } var missing []meta.Connection var assets = make(map[string]meta.Asset) { var list meta.AssetList t.Log("Load datalogger assets file") if err := meta.LoadList("../assets/dataloggers.csv", &list); err != nil { t.Fatal(err) } for _, d := range list { assets[d.Model+":::"+d.Serial] = d } t.Log("Load sensor assets file") if err := meta.LoadList("../assets/sensors.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { assets[s.Model+":::"+s.Serial] = s } } var sensors []meta.InstalledSensor { var list meta.InstalledSensorList t.Log("Load stations file") if err := meta.LoadList("../install/sensors.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { sensors = append(sensors, s) } } for _, s := range sensors { if a, ok := assets[s.Model+":::"+s.Serial]; !ok || a.Number == "" { continue } if s.End.Before(time.Now()) { continue } var handled bool for _, c := range connections { if c.Station != s.Station || c.Location != s.Location { continue } if c.Start.After(s.End) || c.End.Before(s.Start) { continue } handled = true } if !handled { var place string if p, ok := stas[s.Station]; ok { place = p.Name } missing = append(missing, meta.Connection{ Station: s.Station, Location: s.Location, Place: place, Span: meta.Span{ Start: s.Start, End: s.End, }, }) t.Errorf("no current connection defined for sensor: %s [%s/%s] %s %s", s.String(), s.Station, s.Location, s.Start, s.End) } } var dataloggers []meta.DeployedDatalogger { var list meta.DeployedDataloggerList t.Log("Load stations file") if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil { t.Fatal(err) } for _, d := range list { dataloggers = append(dataloggers, d) } } for _, d := range dataloggers { if a, ok := assets[d.Model+":::"+d.Serial]; !ok || a.Number == "" { continue } if d.End.Before(time.Now()) { continue } var handled bool for _, c := range connections { if c.Place != d.Place || c.Role != d.Role { continue } if c.Start.After(d.End) || c.End.Before(d.Start) { continue } handled = true } if !handled { s, l := "XXXX", "LL" for k, v := range stas { if v.Name == d.Place { s = k } } missing = append(missing, meta.Connection{ Station: s, Location: l, Place: d.Place, Role: d.Role, Span: meta.Span{ Start: d.Start, End: d.End, }, }) t.Errorf("no current connection defined for datalogger: %s [%s/%s] %s %s", d.String(), d.Place, d.Role, d.Start, d.End) } } if len(missing) > 0 { sort.Sort(meta.ConnectionList(missing)) t.Log("\n" + string(meta.MarshalList(meta.ConnectionList(missing)))) } }
func TestStreams(t *testing.T) { var streams meta.StreamList t.Log("Load streams file") if err := meta.LoadList("../install/streams.csv", &streams); err != nil { t.Fatal(err) } for i := 0; i < len(streams); i++ { for j := i + 1; j < len(streams); j++ { if streams[i].Station != streams[j].Station { continue } if streams[i].Location != streams[j].Location { continue } if streams[i].Start.After(streams[j].End) { continue } if streams[i].End.Before(streams[j].Start) { continue } if streams[i].SamplingRate != streams[j].SamplingRate { continue } t.Errorf("stream overlap: " + strings.Join([]string{ streams[i].Station, streams[i].Location, streams[i].Start.String(), streams[i].End.String(), }, " ")) } } stas := make(map[string]meta.Station) { var list meta.StationList t.Log("Load stations file") if err := meta.LoadList("../network/stations.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { stas[s.Code] = s } } sites := make(map[string]map[string]meta.Site) { var list meta.SiteList t.Log("Load sites file") if err := meta.LoadList("../network/sites.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { if _, ok := sites[s.Station]; !ok { sites[s.Station] = make(map[string]meta.Site) } sites[s.Station][s.Location] = s } } for _, c := range streams { if _, ok := stas[c.Station]; !ok { t.Log("unknown stream station: " + c.Station) } else if s, ok := sites[c.Station]; !ok { t.Log("unknown stream station: " + c.Station) } else if _, ok := s[c.Location]; !ok { t.Log("unknown stream station/location: " + c.Station + "/" + c.Location) } if c.Start.After(c.End) { t.Log("stream span mismatch: " + strings.Join([]string{ c.Station, c.Location, c.Start.String(), "after", c.End.String(), }, " ")) } } }
func TestAntennas(t *testing.T) { var antennas meta.InstalledAntennaList t.Log("Load deployed antennas file") { if err := meta.LoadList("../install/antennas.csv", &antennas); err != nil { t.Fatal(err) } } t.Log("Check for antenna installation equipment overlaps") { installs := make(map[string]meta.InstalledAntennaList) for _, s := range antennas { if _, ok := installs[s.Model]; !ok { installs[s.Model] = meta.InstalledAntennaList{} } installs[s.Model] = append(installs[s.Model], s) } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("antennas %s [%s] at %s has overlap at %s between %s and %s", v[i].Model, v[i].Serial, v[i].Mark, v[j].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for antenna installation mark overlaps") { installs := make(map[string]meta.InstalledAntennaList) for _, s := range antennas { if _, ok := installs[s.Mark]; !ok { installs[s.Mark] = meta.InstalledAntennaList{} } installs[s.Mark] = append(installs[s.Mark], s) } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("antennas %s [%s] and %s [%s] at %s has overlap between %s and %s", v[i].Model, v[i].Serial, v[j].Model, v[j].Serial, v[i].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for missing antenna marks") { var marks meta.MarkList if err := meta.LoadList("../network/marks.csv", &marks); err != nil { t.Fatal(err) } keys := make(map[string]interface{}) for _, m := range marks { keys[m.Code] = true } for _, c := range antennas { if _, ok := keys[c.Mark]; ok { continue } t.Errorf("unable to find antenna mark %-5s", c.Mark) } } var assets meta.AssetList t.Log("Load antenna assets file") { if err := meta.LoadList("../assets/antennas.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for antenna assets") { for _, r := range antennas { var found bool for _, a := range assets { if a.Model != r.Model { continue } if a.Serial != r.Serial { continue } found = true } if !found { t.Errorf("unable to find antenna asset: %s [%s]", r.Model, r.Serial) } } } var sessions meta.SessionList t.Log("Load session list") { if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil { t.Fatal(err) } } t.Log("Check sessions exist ...") { for _, r := range antennas { var found bool for _, s := range sessions { if s.End.Before(r.Start) { continue } if s.Start.After(r.End) { continue } found = true } if !found { t.Log(r) } } } }
func TestFirmware(t *testing.T) { var firmwares meta.FirmwareHistoryList t.Log("Load firmware history file") { if err := meta.LoadList("../install/firmware.csv", &firmwares); err != nil { t.Fatal(err) } } t.Log("Check for firmware history overlaps") { installs := make(map[string]meta.FirmwareHistoryList) for _, s := range firmwares { _, ok := installs[s.Model] if ok { installs[s.Model] = append(installs[s.Model], s) } else { installs[s.Model] = meta.FirmwareHistoryList{s} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("firmware %s / %s has overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } var assets meta.AssetList t.Log("Load firmware assets file") { if err := meta.LoadList("../assets/receivers.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for firmware receiver assets") { for _, r := range firmwares { var found bool for _, a := range assets { if a.Model != r.Model { continue } if a.Serial != r.Serial { continue } found = true } if !found { t.Errorf("unable to find firmware receiver asset: %s [%s]", r.Model, r.Serial) } } } }
func main() { var verbose bool flag.BoolVar(&verbose, "verbose", false, "make noise") var output string flag.StringVar(&output, "output", "output", "output directory") var logs string flag.StringVar(&logs, "logs", "logs", "logs output directory") var network string flag.StringVar(&network, "network", "../../network", "base network directory") var install string flag.StringVar(&install, "install", "../../install", "base install directory") flag.Usage = func() { fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Build GNSS SiteLog XML files from delta meta information\n") fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Usage:\n") fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, " %s [options]\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Options:\n") fmt.Fprintf(os.Stderr, "\n") flag.PrintDefaults() fmt.Fprintf(os.Stderr, "\n") } flag.Parse() var tplFuncMap template.FuncMap = template.FuncMap{ "empty": func(d, s string) string { if s != "" { return s } return d }, "tolower": func(s string) string { switch t := strings.ToLower(s); t { case "wyatt/agnew drilled-braced": return "Deep Wyatt/Agnew drilled-braced" default: return t } }, "lines": func(p, s string) string { switch s { case "": return s default: return strings.Join(strings.Split(s, "\n"), "\n"+p) } }, "plus": func(n int) string { return fmt.Sprintf("%-2s", strconv.Itoa(n+1)) }, "lat": func(s string) string { if f, err := strconv.ParseFloat(s, 64); err == nil { m := math.Abs(f-float64(int(f))) * 60.0 return fmt.Sprintf("%+3d%02d%05.2f", int(f), int(m), (m-float64(int(m)))*60.0) } return "" }, "lon": func(s string) string { if f, err := strconv.ParseFloat(s, 64); err == nil { m := math.Abs(f-float64(int(f))) * 60.0 return fmt.Sprintf("%+3d%02d%05.2f", int(f), int(m), (m-float64(int(m)))*60.0) } return "" }, } tmpl, err := template.New("").Funcs(tplFuncMap).Parse(sitelogTemplate) if err != nil { log.Fatalf("error: unable to compile template: %v", err) } var firmwareHistoryList meta.FirmwareHistoryList if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil { log.Fatalf("error: unable to load firmware history: %v", err) } firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory) for _, i := range firmwareHistoryList { if _, ok := firmwareHistory[i.Model]; !ok { firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory) } firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i) } for j, _ := range firmwareHistory { for k, _ := range firmwareHistory[j] { sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k])) } } var installedAntennaList meta.InstalledAntennaList if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil { log.Fatalf("error: unable to load antenna installs: %v", err) } installedAntenna := make(map[string][]meta.InstalledAntenna) for _, i := range installedAntennaList { installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i) } for i, _ := range installedAntenna { sort.Sort(meta.InstalledAntennaList(installedAntenna[i])) } var deployedReceiverList meta.DeployedReceiverList if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil { log.Fatalf("error: unable to load receiver installs: %v", err) } deployedReceivers := make(map[string][]meta.DeployedReceiver) for _, i := range deployedReceiverList { deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i) } for i, _ := range deployedReceivers { sort.Sort(meta.DeployedReceiverList(deployedReceivers[i])) } var installedRadomeList meta.InstalledRadomeList if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil { log.Fatalf("error: unable to load radome installs: %v", err) } installedRadomes := make(map[string][]meta.InstalledRadome) for _, i := range installedRadomeList { installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i) } for i, _ := range installedRadomes { sort.Sort(meta.InstalledRadomeList(installedRadomes[i])) } var installedMetSensorList meta.InstalledMetSensorList if err := meta.LoadList(filepath.Join(install, "metsensors.csv"), &installedMetSensorList); err != nil { log.Fatalf("error: unable to load metsensors list: %v", err) } installedMetSensors := make(map[string][]meta.InstalledMetSensor) for _, i := range installedMetSensorList { installedMetSensors[i.Mark] = append(installedMetSensors[i.Mark], i) } for i, _ := range installedMetSensors { sort.Sort(meta.InstalledMetSensorList(installedMetSensors[i])) } var markList meta.MarkList if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil { log.Fatalf("error: unable to load mark list: %v", err) } var sessionList meta.SessionList if err := meta.LoadList(filepath.Join(install, "sessions.csv"), &sessionList); err != nil { log.Fatalf("error: unable to load session list: %v", err) } sessions := make(map[string][]meta.Session) for _, s := range sessionList { sessions[s.Mark] = append(sessions[s.Mark], s) } var monumentList meta.MonumentList if err := meta.LoadList(filepath.Join(network, "monuments.csv"), &monumentList); err != nil { log.Fatalf("error: unable to load monument list: %v", err) } monuments := make(map[string]meta.Monument) for _, m := range monumentList { monuments[m.Mark] = m } for _, m := range markList { if _, ok := monuments[m.Code]; !ok { continue } if _, ok := sessions[m.Code]; !ok { continue } if _, ok := installedAntenna[m.Code]; !ok { continue } if _, ok := deployedReceivers[m.Code]; !ok { continue } var receivers []GnssReceiver var antennas []GnssAntenna var metsensors []GnssMetSensor for _, m := range installedMetSensors[m.Reference.Code] { var session *meta.Session for i, s := range sessions[m.Mark] { if m.Start.After(s.End) || m.End.Before(s.Start) { continue } session = &sessions[m.Mark][i] break } if session == nil { continue } metsensors = append(metsensors, GnssMetSensor{ Manufacturer: m.Make, MetSensorModel: m.Model, SerialNumber: m.Serial, DataSamplingInterval: "360 sec", EffectiveDates: "2000-02-05/CCYY-MM-DD", Notes: "", }) } for _, a := range installedAntenna[m.Code] { var session *meta.Session for i, s := range sessions[m.Code] { if a.Start.After(s.End) || a.End.Before(s.Start) { continue } session = &sessions[m.Code][i] break } if session == nil { continue } radome := "NONE" serial := "" if _, ok := installedRadomes[m.Code]; ok { for _, v := range installedRadomes[m.Code] { if v.Start.After(a.End) || v.End.Before(a.Start) { continue } radome = v.Model serial = v.Serial } } antennas = append(antennas, GnssAntenna{ AntennaType: a.Model, SerialNumber: a.Serial, AntennaReferencePoint: "BAM", MarkerArpUpEcc: strconv.FormatFloat(a.Vertical, 'f', 4, 64), MarkerArpNorthEcc: strconv.FormatFloat(a.North, 'f', 4, 64), MarkerArpEastEcc: strconv.FormatFloat(a.East, 'f', 4, 64), AlignmentFromTrueNorth: "0", AntennaRadomeType: radome, RadomeSerialNumber: serial, AntennaCableType: "", AntennaCableLength: "", DateInstalled: a.Start.Format(DateTimeFormat), DateRemoved: func() string { if time.Now().After(a.End) { return a.End.Format(DateTimeFormat) } else { return "" } }(), Notes: "", }) } for _, r := range deployedReceivers[m.Code] { if _, ok := firmwareHistory[r.Model]; ok { if _, ok := firmwareHistory[r.Model][r.Serial]; ok { for i, _ := range firmwareHistory[r.Model][r.Serial] { v := firmwareHistory[r.Model][r.Serial][len(firmwareHistory[r.Model][r.Serial])-i-1] if v.End.Before(r.Start) || v.Start.After(r.End) { continue } var session *meta.Session for i, s := range sessions[m.Code] { if r.Start.After(s.End) || r.End.Before(s.Start) { continue } if v.Start.After(s.End) || v.End.Before(s.Start) { continue } session = &sessions[m.Code][i] break } if session == nil { continue } start := r.Start /* if start.Before(s.Start) { start = s.Start } */ if start.Before(v.Start) { start = v.Start } end := r.End /* if end.After(s.End) { end = s.End } */ if end.After(v.End) { end = v.End } receivers = append(receivers, GnssReceiver{ ReceiverType: r.Model, SatelliteSystem: session.SatelliteSystem, SerialNumber: r.Serial, FirmwareVersion: v.Version, ElevationCutoffSetting: strconv.FormatFloat(session.ElevationMask, 'g', -1, 64), DateInstalled: start.Format(DateTimeFormat), /* DateInstalled: func() string { if v.Start.Before(r.Start) { return r.Start.Format(DateTimeFormat) } else { return v.Start.Format(DateTimeFormat) } }(), */ DateRemoved: func() string { /* if v.End.After(r.End) { if time.Now().After(r.End) { return r.End.Format(DateTimeFormat) } else { return "" } } else { if time.Now().After(v.End) { return v.End.Format(DateTimeFormat) } else { return "" } } */ if time.Now().After(end) { return end.Format(DateTimeFormat) } else { return "" } }(), TemperatureStabilization: "", Notes: "", }) } } } } sort.Sort(GnssReceivers(receivers)) sort.Sort(GnssAntennas(antennas)) monument := monuments[m.Code] X, Y, Z := WGS842ITRF(m.Latitude, m.Longitude, m.Elevation) x := SiteLog{ EquipNameSpace: equipNameSpace, ContactNameSpace: contactNameSpace, MiNameSpace: miNameSpace, LiNameSpace: liNameSpace, XmlNameSpace: xmlNameSpace, XsiNameSpace: xsiNameSpace, SchemaLocation: schemaLocation, FormInformation: FormInformation{ PreparedBy: preparedBy, DatePrepared: time.Now().Format(DateFormat), ReportType: "DYNAMIC", }, SiteIdentification: SiteIdentification{ SiteName: m.Name, FourCharacterID: m.Code, MonumentInscription: "", IersDOMESNumber: monument.DomesNumber, CdpNumber: "", MonumentDescription: monument.Type, HeightOfTheMonument: strconv.FormatFloat(-monument.GroundRelationship, 'g', -1, 64), MonumentFoundation: monument.FoundationType, FoundationDepth: strconv.FormatFloat(monument.FoundationDepth, 'f', 1, 64), MarkerDescription: func() string { switch monument.MarkType { case "Forced Centering": return "Forced Centering" default: return "unknown" } }(), DateInstalled: m.Start.Format(DateTimeFormat), GeologicCharacteristic: "", BedrockType: "", BedrockCondition: "", FractureSpacing: "", FaultZonesNearby: "", DistanceActivity: "", Notes: "", }, SiteLocation: SiteLocation{ /* City: m.Place, State: m.Region, */ Country: func(lat, lon float64) string { X, Y, _ := WGS842ITRF(lat, lon, 0.0) dist := float64(-1.0) country := "Unknown" for _, v := range countryList { x, y, _ := WGS842ITRF(v.lat, v.lon, 0.0) r := math.Sqrt((x-X)*(x-X) + (y-Y)*(y-Y)) if dist < 0.0 || r < dist { country = v.name dist = r } } return country }(m.Latitude, m.Longitude), TectonicPlate: TectonicPlate(m.Latitude, m.Longitude), ApproximatePositionITRF: ApproximatePositionITRF{ XCoordinateInMeters: strconv.FormatFloat(X, 'f', 1, 64), YCoordinateInMeters: strconv.FormatFloat(Y, 'f', 1, 64), ZCoordinateInMeters: strconv.FormatFloat(Z, 'f', 1, 64), LatitudeNorth: strconv.FormatFloat(m.Latitude, 'g', -1, 64), LongitudeEast: strconv.FormatFloat(m.Longitude, 'g', -1, 64), ElevationMEllips: strconv.FormatFloat(m.Elevation, 'f', 1, 64), }, Notes: "", }, GnssReceivers: receivers, GnssAntennas: antennas, GnssMetSensors: metsensors, ContactAgency: contactAgency, ResponsibleAgency: func() Agency { switch m.Network { case "LI": return responsibleAgency default: return Agency{ MailingAddress: "\n", } } }(), MoreInformation: MoreInformation{ PrimaryDataCenter: primaryDatacentre, SecondaryDataCenter: "", UrlForMoreInformation: urlForMoreInformation, HardCopyOnFile: "", SiteMap: "", SiteDiagram: "", HorizonMask: "", MonumentDescription: "", SitePictures: "", Notes: extraNotes + " " + m.Code, AntennaGraphicsWithDimensions: func() string { var graphs []string models := make(map[string]interface{}) for _, a := range antennas { if _, ok := models[a.AntennaType]; ok { continue } if g, ok := antennaGraphs[a.AntennaType]; ok { b, err := hex.DecodeString(g) if err != nil { log.Printf("error: unable to decode antenna graph for: \"%s\"", a.AntennaType) continue } graphs = append(graphs, strings.Join([]string{a.AntennaType, string(b)}, "\n")) } else { log.Printf("warning: missing antenna graph for: \"%s\"", a.AntennaType) } models[a.AntennaType] = true } return strings.Join(graphs, "\n") + "\n" }(), InsertTextGraphicFromAntenna: "", }, } s, err := x.Marshal() if err != nil { log.Fatalf("error: unable to marshal xml: %v", err) } xmlfile := filepath.Join(output, strings.ToLower(m.Code)+".xml") if err := os.MkdirAll(filepath.Dir(xmlfile), 0755); err != nil { log.Fatalf("error: unable to create dir: %v", err) } if err := ioutil.WriteFile(xmlfile, s, 0644); err != nil { log.Fatalf("error: unable to write file: %v", err) } logfile := filepath.Join(logs, strings.ToLower(m.Code)+".log") if err := os.MkdirAll(filepath.Dir(logfile), 0755); err != nil { log.Fatalf("error: unable to create logs dir: %v", err) } f, err := os.Create(logfile) if err != nil { log.Fatalf("error: unable to create log file: %v", err) } defer f.Close() if err := tmpl.Execute(f, x); err != nil { log.Fatalf("error: unable to write log file: %v", err) } } }
func main() { var verbose bool flag.BoolVar(&verbose, "verbose", false, "make noise") var output string flag.StringVar(&output, "output", "output", "output directory") var network string flag.StringVar(&network, "network", "../../network", "base network directory") var install string flag.StringVar(&install, "install", "../../install", "base install directory") flag.Usage = func() { fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Build a RINEX configuration site XML file from delta meta information\n") fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Usage:\n") fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, " %s [options]\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Options:\n") fmt.Fprintf(os.Stderr, "\n") flag.PrintDefaults() fmt.Fprintf(os.Stderr, "\n") } flag.Parse() var firmwareHistoryList meta.FirmwareHistoryList if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil { fmt.Fprintf(os.Stderr, "error: unable to load firmware history: %v\n", err) os.Exit(-1) } firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory) for _, i := range firmwareHistoryList { if _, ok := firmwareHistory[i.Model]; !ok { firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory) } firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i) } for j, _ := range firmwareHistory { for k, _ := range firmwareHistory[j] { sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k])) } } var installedAntennaList meta.InstalledAntennaList if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil { fmt.Fprintf(os.Stderr, "error: unable to load antenna installs: %v\n", err) os.Exit(-1) } installedAntenna := make(map[string][]meta.InstalledAntenna) for _, i := range installedAntennaList { installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i) } for i, _ := range installedAntenna { sort.Sort(meta.InstalledAntennaList(installedAntenna[i])) } var deployedReceiverList meta.DeployedReceiverList if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil { fmt.Fprintf(os.Stderr, "error: unable to load receiver installs: %v\n", err) os.Exit(-1) } deployedReceivers := make(map[string][]meta.DeployedReceiver) for _, i := range deployedReceiverList { deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i) } for i, _ := range deployedReceivers { sort.Sort(meta.DeployedReceiverList(deployedReceivers[i])) } var installedRadomeList meta.InstalledRadomeList if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil { fmt.Fprintf(os.Stderr, "error: unable to load radome installs: %v\n", err) os.Exit(-1) } installedRadomes := make(map[string][]meta.InstalledRadome) for _, i := range installedRadomeList { installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i) } for i, _ := range installedRadomes { sort.Sort(meta.InstalledRadomeList(installedRadomes[i])) } var markList meta.MarkList if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil { fmt.Fprintf(os.Stderr, "error: unable to load mark list: %v\n", err) os.Exit(-1) } var sessionList meta.SessionList if err := meta.LoadList(filepath.Join(install, "sessions.csv"), &sessionList); err != nil { fmt.Fprintf(os.Stderr, "error: unable to load session list: %v\n", err) os.Exit(-1) } sessions := make(map[string][]meta.Session) for _, s := range sessionList { sessions[s.Mark] = append(sessions[s.Mark], s) } for _, m := range markList { if _, ok := sessions[m.Code]; !ok { continue } if _, ok := installedAntenna[m.Code]; !ok { continue } if _, ok := deployedReceivers[m.Code]; !ok { continue } var list []CGPSSessionXML for _, s := range sessions[m.Code] { for _, a := range installedAntenna[m.Code] { if a.Start.After(s.End) || a.End.Before(s.Start) { continue } if _, ok := IGSModels[a.Model]; !ok { fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s], skipping\n", m.Code, a.Model, a.Serial) continue } for _, r := range deployedReceivers[m.Code] { if r.Start.After(a.End) || r.End.Before(a.Start) { continue } if _, ok := IGSModels[r.Model]; !ok { fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s]", m.Code, r.Model, r.Serial) continue } radome := "NONE" if _, ok := installedRadomes[m.Code]; ok { for _, v := range installedRadomes[m.Code] { if v.Start.After(a.End) || v.Start.After(r.End) { continue } if v.End.Before(a.Start) || v.End.Before(r.Start) { continue } if _, ok := IGSModels[v.Model]; !ok { fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s]", m.Code, v.Model, v.Serial) continue } radome = IGSModels[v.Model] } } var firmware []FirmwareHistoryXML if _, ok := firmwareHistory[r.Model]; ok { if _, ok := firmwareHistory[r.Model][r.Serial]; ok { for i, _ := range firmwareHistory[r.Model][r.Serial] { v := firmwareHistory[r.Model][r.Serial][len(firmwareHistory[r.Model][r.Serial])-i-1] if v.End.Before(r.Start) || v.Start.After(r.End) { continue } firmware = append(firmware, FirmwareHistoryXML{ StartTime: v.Start.Format(meta.DateTimeFormat), StopTime: func() string { if time.Now().After(v.End) { return v.End.Format(meta.DateTimeFormat) } else { return "open" } }(), Version: v.Version, }) } } } list = append(list, CGPSSessionXML{ StartTime: func() string { if r.Start.After(a.Start) { return r.Start.Format(meta.DateTimeFormat) } else if a.Start.After(s.Start) { return a.Start.Format(meta.DateTimeFormat) } else { return s.Start.Format(meta.DateTimeFormat) } }(), StopTime: func() string { if r.End.Before(a.End) { if time.Now().After(r.End) { return r.End.Format(meta.DateTimeFormat) } else { return "open" } } else if a.End.Before(s.End) { if time.Now().After(a.End) { return a.End.Format(meta.DateTimeFormat) } else { return "open" } } else { if time.Now().After(s.End) { return s.End.Format(meta.DateTimeFormat) } else { return "open" } } }(), Receiver: ReceiverXML{ SerialNumber: r.Serial, IGSDesignation: IGSModels[r.Model], FirmwareHistories: firmware, }, InstalledCGPSAntenna: InstalledCGPSAntennaXML{ Height: Number{Units: "m", Value: a.Vertical}, OffsetNorth: Number{Units: "m", Value: a.North}, OffsetEast: Number{Units: "m", Value: a.East}, Radome: radome, CGPSAntenna: CGPSAntennaXML{ SerialNumber: a.Serial, IGSDesignation: IGSModels[a.Model], }, }, ObservationInterval: Number{ Units: "s", Value: s.Interval.Seconds(), }, Operator: OperatorXML{ Name: s.Operator, Agency: s.Agency, }, Rinex: RinexXML{ HeaderCommentName: s.HeaderComment, HeaderCommentText: func() string { if t, ok := HeaderComments[s.HeaderComment]; ok { return strings.Join(strings.Fields(t), " ") } return "" }(), }, DataFormat: func() string { if f, ok := DataFormats[r.Model]; ok { return f } return "unknown" }(), DownloadNameFormat: func() DownloadNameFormatXML { if f, ok := DownloadNameFormats[r.Model]; ok { return f } return DownloadNameFormatXML{} }(), }) } } } x := NewSiteXML( MarkXML{ GeodeticCode: m.Code, //DomesNumber: m.DomesNumber, }, LocationXML{ Latitude: m.Latitude, Longitude: m.Longitude, Height: m.Elevation, Datum: m.Datum, }, list, ) s, err := x.Marshal() if err != nil { fmt.Fprintf(os.Stderr, "error: unable to marsh xml: %v\n", err) os.Exit(-1) } xmlfile := filepath.Join(output, m.Code+".xml") if err := os.MkdirAll(filepath.Dir(xmlfile), 0755); err != nil { fmt.Fprintf(os.Stderr, "error: unable to create dir: %v\n", err) os.Exit(-1) } if err := ioutil.WriteFile(xmlfile, s, 0644); err != nil { fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err) os.Exit(-1) } } }
func TestStreams(t *testing.T) { var streams meta.StreamList t.Log("Load streams file") if err := meta.LoadList("../install/streams.csv", &streams); err != nil { t.Fatal(err) } for i := 0; i < len(streams); i++ { for j := i + 1; j < len(streams); j++ { if streams[i].Station != streams[j].Station { continue } if streams[i].Location != streams[j].Location { continue } if streams[i].Start.After(streams[j].End) { continue } if streams[i].End.Before(streams[j].Start) { continue } if streams[i].SamplingRate != streams[j].SamplingRate { continue } t.Errorf("stream overlap: " + strings.Join([]string{ streams[i].Station, streams[i].Location, streams[i].Start.String(), streams[i].End.String(), }, " ")) } } stas := make(map[string]meta.Station) { var list meta.StationList t.Log("Load stations file") if err := meta.LoadList("../network/stations.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { stas[s.Code] = s } } sites := make(map[string]map[string]meta.Site) { var list meta.SiteList t.Log("Load sites file") if err := meta.LoadList("../network/sites.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { if _, ok := sites[s.Station]; !ok { sites[s.Station] = make(map[string]meta.Site) } sites[s.Station][s.Location] = s } } for _, s := range streams { if s.SamplingRate == 0 { t.Errorf("invalid stream sample rate: " + strings.Join([]string{ s.Station, s.Location, s.Start.String(), s.End.String(), }, " ")) } } for _, c := range streams { if _, ok := stas[c.Station]; !ok { t.Log("unknown stream station: " + c.Station) } else if s, ok := sites[c.Station]; !ok { t.Log("unknown stream station: " + c.Station) } else if _, ok := s[c.Location]; !ok { t.Log("unknown stream station/location: " + c.Station + "/" + c.Location) } if c.Start.After(c.End) { t.Log("stream span mismatch: " + strings.Join([]string{ c.Station, c.Location, c.Start.String(), "after", c.End.String(), }, " ")) } } var assets = make(map[string]meta.Asset) { var list meta.AssetList t.Log("Load recorders assets file") if err := meta.LoadList("../assets/recorders.csv", &list); err != nil { t.Fatal(err) } for _, l := range list { assets[l.Model+":::"+l.Serial] = l } t.Log("Load sensors assets file") if err := meta.LoadList("../assets/sensors.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { assets[s.Model+":::"+s.Serial] = s } } var recorders []meta.InstalledRecorder { var list meta.InstalledRecorderList t.Log("Load recorders file") if err := meta.LoadList("../install/recorders.csv", &list); err != nil { t.Fatal(err) } for _, r := range list { recorders = append(recorders, r) } } var missing []meta.Stream for _, r := range recorders { if a, ok := assets[r.Model+":::"+r.Serial]; !ok || a.Number == "" { continue } if r.End.Before(time.Now()) { continue } var handled bool for _, s := range streams { if s.Station != r.Station || r.Location != s.Location { continue } if r.Start.After(s.End) || r.End.Before(s.Start) { continue } handled = true } if !handled { for _, sps := range recorderSamplingRates { missing = append(missing, meta.Stream{ Station: r.Station, Location: r.Location, SamplingRate: sps, Span: meta.Span{ Start: r.Start, End: r.End, }, }) } t.Errorf("no current stream defined for recorder: %s [%s/%s] %s %s", r.String(), r.Station, r.Location, r.Start, r.End) } } var sensors []meta.InstalledSensor { var list meta.InstalledSensorList t.Log("Load sensors file") if err := meta.LoadList("../install/sensors.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { sensors = append(sensors, s) } } for _, v := range sensors { if a, ok := assets[v.Model+":::"+v.Serial]; !ok || a.Number == "" { continue } if v.End.Before(time.Now()) { continue } var handled bool for _, s := range streams { if s.Station != v.Station || v.Location != s.Location { continue } if v.Start.After(s.End) || v.End.Before(s.Start) { continue } handled = true } if !handled { for _, sps := range sensorSamplingRates { missing = append(missing, meta.Stream{ Station: v.Station, Location: v.Location, SamplingRate: sps, Span: meta.Span{ Start: v.Start, End: v.End, }, }) } t.Errorf("no current stream defined for sensor: %s [%s/%s] %s %s", v.String(), v.Station, v.Location, v.Start, v.End) } } if len(missing) > 0 { sort.Sort(meta.StreamList(missing)) t.Log("\n" + string(meta.MarshalList(meta.StreamList(missing)))) } }
// TestMarksProto creates a Protobuf file of Marks. func TestMarksProto(t *testing.T) { var networks meta.NetworkList if err := meta.LoadList("../network/networks.csv", &networks); err != nil { t.Error(err) } var net = make(map[string]*delta.Network) for _, v := range networks { n := delta.Network{ Code: v.Code, External: v.External, Description: v.Description, Restricted: v.Restricted, } net[v.Code] = &n } var marks meta.MarkList if err := meta.LoadList("../network/marks.csv", &marks); err != nil { t.Error(err) } if len(marks) == 0 { t.Error("zero length mark list.") } var m delta.Marks m.Marks = make(map[string]*delta.Mark) for _, v := range marks { pt := delta.Point{ Longitude: v.Longitude, Latitude: v.Latitude, Elevation: v.Elevation, Datum: v.Datum, } s := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } mk := delta.Mark{ Code: v.Code, Name: v.Name, Network: net[v.Network], Point: &pt, Span: &s, } m.Marks[mk.Code] = &mk } b, err := proto.Marshal(&m) if err != nil { t.Error(err) } if err := os.MkdirAll(apiDir, 0777); err != nil { t.Error(err) } if err := ioutil.WriteFile(apiDir+"/marks.pb", b, 0644); err != nil { t.Error(err) } }
func TestSessions(t *testing.T) { var sessions meta.SessionList t.Log("Load sessions file") if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil { t.Fatal(err) } for i := 0; i < len(sessions); i++ { for j := i + 1; j < len(sessions); j++ { if sessions[i].Mark != sessions[j].Mark { continue } if sessions[i].Model != sessions[j].Model { continue } if sessions[i].Interval != sessions[j].Interval { continue } if sessions[i].Start.After(sessions[j].End) { continue } if !sessions[i].End.After(sessions[j].Start) { continue } t.Errorf("session overlap: " + strings.Join([]string{ sessions[i].Mark, sessions[i].Interval.String(), sessions[i].Start.String(), sessions[i].End.String(), sessions[j].Interval.String(), sessions[j].Start.String(), sessions[j].End.String(), }, " ")) } } marks := make(map[string]meta.Mark) { var list meta.MarkList t.Log("Load marks file") if err := meta.LoadList("../network/marks.csv", &list); err != nil { t.Fatal(err) } for _, m := range list { marks[m.Code] = m } } for _, s := range sessions { if _, ok := marks[s.Mark]; !ok { t.Log("unknown session mark: " + s.Mark) } if s.Start.After(s.End) { t.Log("session span mismatch: " + strings.Join([]string{ s.Mark, s.Interval.String(), s.Start.String(), "after", s.End.String(), }, " ")) } switch s.SatelliteSystem { case "GPS": case "GPS+GLO": case "GPS+GLO+GAL+BDS+QZSS": default: t.Error("unknown satellite system: " + s.SatelliteSystem) } } }
func TestCombined(t *testing.T) { var combined meta.InstalledSensorList t.Log("Load installed sensors file") { if err := meta.LoadList("../install/sensors.csv", &combined); err != nil { t.Fatal(err) } } var recorders meta.InstalledRecorderList t.Log("Load installed recorders file") { if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil { t.Fatal(err) } } for _, r := range recorders { combined = append(combined, meta.InstalledSensor{ Install: r.Install, Station: r.Station, Location: r.Location, }) } t.Log("Check for sensor/recorder installation location overlaps") { installs := make(map[string]meta.InstalledSensorList) for _, s := range combined { _, ok := installs[s.Station] if ok { installs[s.Station] = append(installs[s.Station], s) } else { installs[s.Station] = meta.InstalledSensorList{s} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Location != v[j].Location: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("sensor/recorder %s/%s at %-5s has location %-2s overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } }
// TestStationsProto creates: // Protobuf file of Stations // JSON file of Stations // GeoJSON of Site for each sensor type. func TestStationsProto(t *testing.T) { var networks meta.NetworkList if err := meta.LoadList("../network/networks.csv", &networks); err != nil { t.Error(err) } var net = make(map[string]*delta.Network) for _, v := range networks { n := delta.Network{ Code: v.Code, External: v.External, Description: v.Description, Restricted: v.Restricted, } net[v.Code] = &n } var stations meta.StationList if err := meta.LoadList("../network/stations.csv", &stations); err != nil { t.Error(err) } if len(stations) == 0 { t.Error("zero length stations list.") } var s delta.Stations s.Stations = make(map[string]*delta.Station) for _, v := range stations { pt := delta.Point{ Longitude: v.Longitude, Latitude: v.Latitude, Elevation: v.Elevation, Datum: v.Datum, } sp := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } st := delta.Station{ Code: v.Code, Name: v.Name, Network: net[v.Network], Point: &pt, Span: &sp, } s.Stations[st.Code] = &st } var sites meta.SiteList if err := meta.LoadList("../network/sites.csv", &sites); err != nil { t.Error(err) } for _, v := range sites { pt := delta.Point{ Longitude: v.Longitude, Latitude: v.Latitude, Elevation: v.Elevation, Datum: v.Datum, } sp := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } si := delta.Site{ Location: v.Location, Survey: v.Survey, Span: &sp, Point: &pt, } if _, ok := s.Stations[v.Station]; ok { if s.Stations[v.Station].Sites == nil { s.Stations[v.Station].Sites = make(map[string]*delta.Site) } s.Stations[v.Station].Sites[si.Location] = &si } } var installed meta.InstalledSensorList if err := meta.LoadList("../install/sensors.csv", &installed); err != nil { t.Fatal(err) } for _, v := range installed { st := resp.SensorModels[v.Model] e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, Type: st.Type, } sp := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } o := delta.Orientation{ Dip: v.Dip, Azimuth: v.Azimuth, } off := delta.Offset{ North: v.North, East: v.East, Vertical: v.Vertical, } sc := delta.Scale{ Bias: v.Bias, Factor: v.Factor, } is := delta.InstalledSensor{ Equipment: &e, Span: &sp, Orientation: &o, Offset: &off, Scale: &sc, } if _, ok := s.Stations[v.Station]; ok { if _, ok := s.Stations[v.Station].Sites[v.Location]; ok { s.Stations[v.Station].Sites[v.Location].InstalledSensor = append(s.Stations[v.Station].Sites[v.Location].InstalledSensor, &is) } } } // strong motion recorders that are a sensor and a datalogger as a package. var recorders meta.InstalledRecorderList if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil { t.Fatal(err) } for _, v := range recorders { st, ok := resp.SensorModels[v.InstalledSensor.Model] if !ok { // in resp/auto.go some models have " SENSOR" appended. st, ok = resp.SensorModels[v.InstalledSensor.Model+" SENSOR"] } e := delta.Equipment{ Make: v.Make, Model: v.Model, Serial: v.Serial, Type: st.Type, } sp := delta.Span{ Start: v.Start.Unix(), End: v.End.Unix(), } o := delta.Orientation{ Dip: v.Dip, Azimuth: v.Azimuth, } off := delta.Offset{ North: v.North, East: v.East, Vertical: v.Vertical, } sc := delta.Scale{ Bias: v.Bias, Factor: v.Factor, } is := delta.InstalledSensor{ Equipment: &e, Span: &sp, Orientation: &o, Offset: &off, Scale: &sc, } if _, ok := s.Stations[v.Station]; ok { if _, ok := s.Stations[v.Station].Sites[v.Location]; ok { s.Stations[v.Station].Sites[v.Location].InstalledSensor = append(s.Stations[v.Station].Sites[v.Location].InstalledSensor, &is) } } } // protobuf of all station information b, err := proto.Marshal(&s) if err != nil { t.Error(err) } if err := os.MkdirAll(apiDir, 0777); err != nil { t.Error(err) } if err := ioutil.WriteFile(apiDir+"/stations.pb", b, 0644); err != nil { t.Error(err) } // json of all station information b, err = json.Marshal(&s) if err != nil { t.Error(err) } if err := os.MkdirAll(apiDir, 0777); err != nil { t.Error(err) } if err := ioutil.WriteFile(apiDir+"/stations.json", b, 0644); err != nil { t.Error(err) } // GeoJSON files of site for each sensor type. var out = make(map[string]*bytes.Buffer) for _, v := range resp.SensorModels { if v.Type != "" { out[v.Type] = &bytes.Buffer{} } } for k := range out { out[k].WriteString(`{"type": "FeatureCollection","features": [`) } for _, station := range s.Stations { if station.GetSpan() == nil || station.GetPoint() == nil || station.GetNetwork() == nil || station.Network.Restricted { continue } if station.Network.External != "NZ" || station.Network.Code == "XX" { continue } for _, site := range station.Sites { for _, v := range site.InstalledSensor { _, ok := out[v.Equipment.Type] switch { case v.Equipment.Type == "": t.Logf("%s.%s no sensor model for %s, site not classified.", station.Code, site.Location, v.Equipment.Model) case ok: writeProps(station, site, v, out[v.Equipment.Type]) case !ok: t.Logf("unclassified type %s", v.Equipment.Type) } } } } for k := range out { out[k].WriteString(`]}`) fn := strings.Replace(strings.ToLower(k), " ", "", -1) fn = apiDir + "/" + fn + ".geojson" if err := ioutil.WriteFile(fn, out[k].Bytes(), 0644); err != nil { t.Error(err) } } }
func TestDataloggers(t *testing.T) { var dataloggers meta.DeployedDataloggerList t.Log("Load deployed dataloggers file") { if err := meta.LoadList("../install/dataloggers.csv", &dataloggers); err != nil { t.Fatal(err) } } t.Log("Check for datalogger installation place overlaps") { installs := make(map[string]meta.DeployedDataloggerList) for _, d := range dataloggers { _, ok := installs[d.Place] if ok { installs[d.Place] = append(installs[d.Place], d) } else { installs[d.Place] = meta.DeployedDataloggerList{d} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Place != v[j].Place: case v[i].Role != v[j].Role: //case v[i].Model != v[j].Model: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): //case v[i].End.Equal(v[j].Start): //case v[i].Start.Equal(v[j].End): default: t.Errorf("datalogger %s:[%s] at %-32s has place overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Place, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for datalogger installation equipment overlaps") { installs := make(map[string]meta.DeployedDataloggerList) for _, s := range dataloggers { _, ok := installs[s.Model] if ok { installs[s.Model] = append(installs[s.Model], s) } else { installs[s.Model] = meta.DeployedDataloggerList{s} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): // case v[i].End.Equal(v[j].Start): // case v[i].Start.Equal(v[j].End): default: t.Errorf("datalogger %s:[%s] at %-32s has installation overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Place, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } var assets meta.AssetList t.Log("Load datalogger assets file") { if err := meta.LoadList("../assets/dataloggers.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for datalogger assets") { for _, r := range dataloggers { var found bool for _, a := range assets { if a.Model != r.Model { continue } if a.Serial != r.Serial { continue } found = true } if !found { t.Errorf("unable to find datalogger asset: %s [%s]", r.Model, r.Serial) } } } }
func NewMetaDB(base string) (*MetaDB, error) { db, err := memdb.NewMemDB(NewSchema()) if err != nil { return nil, err } var networks = []struct { table string path string list meta.List }{ {"gauge", "network/gauges.csv", &meta.GaugeList{}}, {"constituent", "network/constituents.csv", &meta.ConstituentList{}}, {"mark", "network/marks.csv", &meta.MarkList{}}, {"monument", "network/monuments.csv", &meta.MonumentList{}}, {"mount", "network/mounts.csv", &meta.MountList{}}, {"network", "network/networks.csv", &meta.NetworkList{}}, {"site", "network/sites.csv", &meta.SiteList{}}, {"station", "network/stations.csv", &meta.StationList{}}, } txn := db.Txn(true) for _, list := range networks { if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) { continue } switch list.list.(type) { case *meta.ConstituentList: var input meta.ConstituentList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.GaugeList: var input meta.GaugeList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.MarkList: var input meta.MarkList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.MonumentList: var input meta.MonumentList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.MountList: var input meta.MountList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.NetworkList: var input meta.NetworkList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.SiteList: var input meta.SiteList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.StationList: var input meta.StationList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } default: return nil, fmt.Errorf("invalid type found for: %s", list.path) } } txn.Commit() var assets = []struct { table string path string }{ {"antenna", "assets/antennas.csv"}, {"camera", "assets/cameras.csv"}, {"datalogger", "assets/dataloggers.csv"}, {"metsensor", "assets/metsensors.csv"}, {"radome", "assets/radomes.csv"}, {"receiver", "assets/receivers.csv"}, {"recorder", "assets/recorders.csv"}, {"sensor", "assets/sensors.csv"}, } txn = db.Txn(true) for _, list := range assets { if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) { continue } var input meta.AssetList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert("asset", i); err != nil { return nil, err } } } txn.Commit() var installs = []struct { table string path string list meta.List }{ {"antenna", "install/antennas.csv", &meta.InstalledAntennaList{}}, {"camera", "install/cameras.csv", &meta.InstalledCameraList{}}, {"datalogger", "install/dataloggers.csv", &meta.DeployedDataloggerList{}}, {"metsensor", "install/metsensors.csv", &meta.InstalledMetSensorList{}}, {"radome", "install/radomes.csv", &meta.InstalledRadomeList{}}, {"receiver", "install/receivers.csv", &meta.DeployedReceiverList{}}, {"recorder", "install/recorders.csv", &meta.InstalledRecorderList{}}, {"sensor", "install/sensors.csv", &meta.InstalledSensorList{}}, {"connection", "install/connections.csv", &meta.ConnectionList{}}, {"firmware", "install/firmware.csv", &meta.FirmwareHistoryList{}}, {"session", "install/sessions.csv", &meta.SessionList{}}, {"stream", "install/streams.csv", &meta.StreamList{}}, } txn = db.Txn(true) for _, list := range installs { if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) { continue } switch list.list.(type) { case *meta.InstalledAntennaList: var input meta.InstalledAntennaList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.InstalledCameraList: var input meta.InstalledCameraList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.DeployedDataloggerList: var input meta.DeployedDataloggerList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.InstalledMetSensorList: var input meta.InstalledMetSensorList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.InstalledRadomeList: var input meta.InstalledRadomeList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.DeployedReceiverList: var input meta.DeployedReceiverList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.InstalledRecorderList: var input meta.InstalledRecorderList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.InstalledSensorList: var input meta.InstalledSensorList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.ConnectionList: var input meta.ConnectionList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.FirmwareHistoryList: var input meta.FirmwareHistoryList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.SessionList: var input meta.SessionList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } case *meta.StreamList: var input meta.StreamList if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil { return nil, err } for _, i := range input { if err := txn.Insert(list.table, i); err != nil { return nil, err } } default: return nil, fmt.Errorf("invalid type found for: %s", list.path) } } txn.Commit() return &MetaDB{MemDB: db}, nil }
func TestConsistency(t *testing.T) { files := map[string]struct { f string l meta.List }{ "connections": {f: "../install/connections.csv", l: &meta.ConnectionList{}}, "cameras": {f: "../install/cameras.csv", l: &meta.InstalledCameraList{}}, "dataloggers": {f: "../install/dataloggers.csv", l: &meta.DeployedDataloggerList{}}, "metsensors": {f: "../install/metsensors.csv", l: &meta.InstalledMetSensorList{}}, "radomes": {f: "../install/radomes.csv", l: &meta.InstalledRadomeList{}}, "receivers": {f: "../install/receivers.csv", l: &meta.DeployedReceiverList{}}, "recorders": {f: "../install/recorders.csv", l: &meta.InstalledRecorderList{}}, "sensors": {f: "../install/sensors.csv", l: &meta.InstalledSensorList{}}, "firmware": {f: "../install/firmware.csv", l: &meta.FirmwareHistoryList{}}, "streams": {f: "../install/streams.csv", l: &meta.StreamList{}}, "networks": {f: "../network/networks.csv", l: &meta.NetworkList{}}, "stations": {f: "../network/stations.csv", l: &meta.StationList{}}, "sites": {f: "../network/sites.csv", l: &meta.SiteList{}}, "marks": {f: "../network/marks.csv", l: &meta.MarkList{}}, "mounts": {f: "../network/mounts.csv", l: &meta.MountList{}}, "gauges": {f: "../network/gauges.csv", l: &meta.GaugeList{}}, "constituents": {f: "../network/constituents.csv", l: &meta.ConstituentList{}}, } for k, v := range files { if err := meta.LoadList(v.f, v.l); err != nil { t.Fatal(err) } sort.Sort(v.l) raw, err := ioutil.ReadFile(v.f) if err != nil { t.Fatal(err) } var buf bytes.Buffer if err := csv.NewWriter(&buf).WriteAll(meta.EncodeList(v.l)); err != nil { t.Fatal(err) } if string(raw) != buf.String() { t.Error(k + ": **** csv file mismatch **** : " + v.f) file, err := ioutil.TempFile(os.TempDir(), "tst") if err != nil { t.Fatal(err) } defer os.Remove(file.Name()) file.Write(buf.Bytes()) cmd := exec.Command("diff", "-c", v.f, file.Name()) stdout, err := cmd.StdoutPipe() if err != nil { t.Fatal(err) } err = cmd.Start() if err != nil { t.Fatal(err) } defer cmd.Wait() diff, err := ioutil.ReadAll(stdout) if err != nil { t.Fatal(err) } t.Error(string(diff)) } } }
func NewMeta(network, install string) (*Meta, error) { networkMap := make(map[string]meta.Network) var n meta.NetworkList if err := meta.LoadList(filepath.Join(network, "networks.csv"), &n); err != nil { return nil, err } for _, v := range n { networkMap[v.Code] = v } stationMap := make(map[string]meta.Station) var stations meta.StationList if err := meta.LoadList(filepath.Join(network, "stations.csv"), &stations); err != nil { return nil, err } for _, v := range stations { stationMap[v.Code] = v } connectionMap := make(map[string]meta.ConnectionList) var connections meta.ConnectionList if err := meta.LoadList(filepath.Join(install, "connections.csv"), &connections); err != nil { return nil, err } for _, c := range connections { if _, ok := connectionMap[c.Station]; ok { connectionMap[c.Station] = append(connectionMap[c.Station], c) } else { connectionMap[c.Station] = meta.ConnectionList{c} } } var recorders meta.InstalledRecorderList if err := meta.LoadList(filepath.Join(install, "recorders.csv"), &recorders); err != nil { return nil, err } for _, r := range recorders { c := meta.Connection{ Station: r.Station, Location: r.Location, Span: meta.Span{ Start: r.Start, End: r.End, }, Place: r.Station, Role: r.Location, } if _, ok := connectionMap[c.Station]; ok { connectionMap[c.Station] = append(connectionMap[c.Station], c) } else { connectionMap[c.Station] = meta.ConnectionList{c} } } siteMap := make(map[string]map[string]meta.Site) var locations meta.SiteList if err := meta.LoadList(filepath.Join(network, "sites.csv"), &locations); err != nil { return nil, err } for _, l := range locations { if _, ok := siteMap[l.Station]; !ok { siteMap[l.Station] = make(map[string]meta.Site) } siteMap[l.Station][l.Location] = l } streamMap := make(map[string]map[string]meta.StreamList) var streams meta.StreamList if err := meta.LoadList(filepath.Join(install, "streams.csv"), &streams); err != nil { return nil, err } for _, s := range streams { if _, ok := streamMap[s.Station]; !ok { streamMap[s.Station] = make(map[string]meta.StreamList) } if _, ok := streamMap[s.Station][s.Location]; ok { streamMap[s.Station][s.Location] = append(streamMap[s.Station][s.Location], s) } else { streamMap[s.Station][s.Location] = meta.StreamList{s} } } sensorInstalls := make(map[string]meta.InstalledSensorList) // build sensor installation details var sensors meta.InstalledSensorList if err := meta.LoadList(filepath.Join(install, "sensors.csv"), &sensors); err != nil { return nil, err } for _, s := range sensors { if _, ok := sensorInstalls[s.Station]; ok { sensorInstalls[s.Station] = append(sensorInstalls[s.Station], s) } else { sensorInstalls[s.Station] = meta.InstalledSensorList{s} } } for _, r := range recorders { if _, ok := sensorInstalls[r.Station]; ok { sensorInstalls[r.Station] = append(sensorInstalls[r.Station], r.InstalledSensor) } else { sensorInstalls[r.Station] = meta.InstalledSensorList{r.InstalledSensor} } } for i, _ := range sensorInstalls { sort.Sort(sensorInstalls[i]) } dataloggerDeploys := make(map[string]meta.DeployedDataloggerList) // where the dataloggers were deployed var loggers meta.DeployedDataloggerList if err := meta.LoadList(filepath.Join(install, "dataloggers.csv"), &loggers); err != nil { return nil, err } for _, d := range loggers { if _, ok := dataloggerDeploys[d.Place]; ok { dataloggerDeploys[d.Place] = append(dataloggerDeploys[d.Place], d) } else { dataloggerDeploys[d.Place] = meta.DeployedDataloggerList{d} } } for _, r := range recorders { d := meta.DeployedDatalogger{ Install: meta.Install{ Equipment: meta.Equipment{ Make: r.Make, Model: r.DataloggerModel, Serial: r.Serial, }, Span: meta.Span{ Start: r.Start, End: r.End, }, }, Place: r.Station, Role: r.Location, } if _, ok := dataloggerDeploys[d.Place]; ok { dataloggerDeploys[d.Place] = append(dataloggerDeploys[d.Place], d) } else { dataloggerDeploys[d.Place] = meta.DeployedDataloggerList{d} } } // sort each datalogger deployment for i, _ := range dataloggerDeploys { sort.Sort(dataloggerDeploys[i]) } return &Meta{ Networks: networkMap, Stations: stationMap, Connections: connectionMap, Sites: siteMap, Streams: streamMap, Installs: sensorInstalls, Deploys: dataloggerDeploys, }, nil }
func TestCameras(t *testing.T) { var cameras meta.InstalledCameraList t.Log("Load deployed cameras file") { if err := meta.LoadList("../install/cameras.csv", &cameras); err != nil { t.Fatal(err) } } t.Log("Check for cameras installation equipment overlaps") { installs := make(map[string]meta.InstalledCameraList) for _, s := range cameras { _, ok := installs[s.Model] if ok { installs[s.Model] = append(installs[s.Model], s) } else { installs[s.Model] = meta.InstalledCameraList{s} } } var keys []string for k, _ := range installs { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := installs[k] for i, n := 0, len(v); i < n; i++ { for j := i + 1; j < n; j++ { switch { case v[i].Serial != v[j].Serial: case v[i].End.Before(v[j].Start): case v[i].Start.After(v[j].End): case v[i].End.Equal(v[j].Start): case v[i].Start.Equal(v[j].End): default: t.Errorf("cameras %s at %-5s has mount %s overlap between %s and %s", v[i].Model, v[i].Serial, v[i].Mount, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat)) } } } } } t.Log("Check for missing camera mounts") { var mounts meta.MountList if err := meta.LoadList("../network/mounts.csv", &mounts); err != nil { t.Fatal(err) } keys := make(map[string]interface{}) for _, m := range mounts { keys[m.Code] = true } for _, c := range cameras { if _, ok := keys[c.Mount]; ok { continue } t.Errorf("unable to find camera mount %-5s", c.Mount) } } var assets meta.AssetList t.Log("Load camera assets file") { if err := meta.LoadList("../assets/cameras.csv", &assets); err != nil { t.Fatal(err) } } t.Log("Check for camera assets") { for _, r := range cameras { var found bool for _, a := range assets { if a.Model != r.Model { continue } if a.Serial != r.Serial { continue } found = true } if !found { t.Errorf("unable to find camera asset: %s [%s]", r.Model, r.Serial) } } } }
func TestConnections(t *testing.T) { var connections meta.ConnectionList t.Log("Load connections file") if err := meta.LoadList("../install/connections.csv", &connections); err != nil { t.Fatal(err) } for i := 0; i < len(connections); i++ { for j := i + 1; j < len(connections); j++ { if connections[i].Station != connections[j].Station { continue } if connections[i].Location != connections[j].Location { continue } if connections[i].Start.After(connections[j].End) { continue } if connections[i].End.Before(connections[j].Start) { continue } t.Errorf("connection overlap: " + strings.Join([]string{ connections[i].Station, connections[i].Location, connections[i].Start.String(), connections[i].End.String(), }, " ")) } } stas := make(map[string]meta.Station) { var list meta.StationList t.Log("Load stations file") if err := meta.LoadList("../network/stations.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { stas[s.Code] = s } } sites := make(map[string]map[string]meta.Site) { var list meta.SiteList t.Log("Load sites file") if err := meta.LoadList("../network/sites.csv", &list); err != nil { t.Fatal(err) } for _, s := range list { if _, ok := sites[s.Station]; !ok { sites[s.Station] = make(map[string]meta.Site) } sites[s.Station][s.Location] = s } } for _, c := range connections { if _, ok := stas[c.Station]; !ok { t.Log("unknown connection station: " + c.Station) } else if s, ok := sites[c.Station]; !ok { t.Log("unknown connection station: " + c.Station) } else if _, ok := s[c.Location]; !ok { t.Log("unknown connection station/location: " + c.Station + "/" + c.Location) } if c.Start.After(c.End) { t.Log("connection span mismatch: " + strings.Join([]string{ c.Station, c.Location, c.Start.String(), "after", c.End.String(), }, " ")) } } places := make(map[string]string) { var list meta.DeployedDataloggerList t.Log("Load installed dataloggers file") if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil { t.Fatal(err) } for _, d := range list { if d.Role != "" { places[d.Place+"/"+d.Role] = d.Place } else { places[d.Place] = d.Place } } } for _, c := range connections { if c.Role != "" { if _, ok := places[c.Place+"/"+c.Role]; !ok { t.Log("warning: unknown datalogger place/role: " + c.Place + "/" + c.Role) } } else { if _, ok := places[c.Place]; !ok { t.Log("warning: unknown datalogger place: " + c.Place) } } } /* Station Code,Location Code,Datalogger Place,Datalogger Role,Pre Amp,Gain,Start Date,End Date A11,10,Alfredton,Short Period,false,0,1993-02-01T01:23:00Z,1993-02-21T08:48:00Z ABAZ,10,Whangaparaoa Navy Base,,false,0,2008-10-13T04:00:00Z,9999-01-01T00:00:00Z AC1A,10,Wards Pass #1,,false,0,2001-09-12T03:00:01Z,2002-01-22T22:00:00Z AC2A,10,Wards Pass #2,,false,0,2001-09-13T02:00:01Z,2002-01-22T22:00:00Z AC3A,10,Acheron #3,,false,0,2001-07-31T23:00:00Z,2002-01-22T23:00:00Z AC4A,10,Acheron #4,Short Period,false,0,2001-08-01T01:00:00Z,2002-01-22T23:00:00Z AC5A,10,Wards Pass #5,,false,0,2001-08-01T04:00:00Z,2001-11-27T23:00:00Z AGA,10,Angora Road,Short Period,false,0,1990-02-20T22:33:00Z,1992-04-07T17:01:00Z AHAA,10,Ahaura,Short Period,false,0,1995-11-19T17:31:00Z,1995-12-12T01:28:00Z */ /* */ }