func TestConstituents(t *testing.T) {
gauges := make(map[string]meta.Gauge)
{
t.Log("Load installed gauges file")
var list meta.GaugeList
if err := meta.LoadList("../network/gauges.csv", &list); err != nil {
t.Fatal(err)
}
for _, g := range list {
gauges[g.Code] = g
}
}
var constituents meta.ConstituentList
if err := meta.LoadList("../network/constituents.csv", &constituents); err != nil {
t.Fatal(err)
}
for i := 0; i < len(constituents); i++ {
for j := i + 1; j < len(constituents); j++ {
if constituents[i].Gauge == constituents[j].Gauge && constituents[i].Number == constituents[j].Number {
t.Error("contituent duplication: " + constituents[i].Gauge + "/" + strconv.Itoa(constituents[i].Number))
}
}
}
for _, c := range constituents {
if _, ok := gauges[c.Gauge]; !ok {
t.Error("unknown gauge: " + c.Gauge)
}
}
}
func TestMonuments(t *testing.T) {
var monuments meta.MonumentList
t.Log("Load network monuments file")
{
if err := meta.LoadList("../network/monuments.csv", &monuments); err != nil {
t.Fatal(err)
}
}
for i := 0; i < len(monuments); i++ {
for j := i + 1; j < len(monuments); j++ {
if monuments[i].Mark == monuments[j].Mark {
t.Errorf("monument duplication: " + monuments[i].Mark)
}
}
}
for _, m := range monuments {
if m.GroundRelationship > 0.0 {
t.Errorf("positive monuments ground relationship: %s [%g]", m.Mark, m.GroundRelationship)
}
switch m.Type {
case "Shallow Rod / Braced Antenna Mount":
case "Wyatt/Agnew Drilled-Braced":
case "Pillar":
case "Steel Mast":
case "Unknown":
default:
t.Errorf("unknown monument type: %s [%s]", m.Mark, m.Type)
}
}
}
func TestGauges(t *testing.T) {
stas := make(map[string]meta.Station)
{
var list meta.StationList
t.Log("Load stations file")
if err := meta.LoadList("../network/stations.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
stas[s.Code] = s
}
}
var gauges meta.GaugeList
t.Log("Load installed gauges file")
{
if err := meta.LoadList("../network/gauges.csv", &gauges); err != nil {
t.Fatal(err)
}
}
for i := 0; i < len(gauges); i++ {
for j := i + 1; j < len(gauges); j++ {
if gauges[i].Code == gauges[j].Code {
t.Errorf("gauge code duplication: " + gauges[i].Code)
}
if gauges[i].Number == gauges[j].Number {
t.Errorf("gauge number duplication: " + gauges[i].Code)
}
}
}
for _, g := range gauges {
if _, ok := stas[g.Code]; !ok {
t.Error("unknown gauge station: " + g.Code)
}
}
}
func TestSites(t *testing.T) {
var sites meta.SiteList
t.Log("Load installed sites file")
{
if err := meta.LoadList("../network/sites.csv", &sites); err != nil {
t.Fatal(err)
}
}
for i := 0; i < len(sites); i++ {
for j := i + 1; j < len(sites); j++ {
if sites[i].Station == sites[j].Station && sites[i].Location == sites[j].Location {
t.Errorf("site duplication: " + sites[i].Station + "/" + sites[i].Location)
}
}
}
}
func TestNetworks(t *testing.T) {
var networks meta.NetworkList
t.Log("Load installed sensors file")
{
if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
t.Fatal(err)
}
}
for i := 0; i < len(networks); i++ {
for j := i + 1; j < len(networks); j++ {
if networks[i].Code == networks[j].Code {
t.Errorf("network duplication: " + networks[i].Code)
}
}
}
}
func TestStations(t *testing.T) {
var stations meta.StationList
t.Log("Load installed sensors file")
{
if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
t.Fatal(err)
}
}
for i := 0; i < len(stations); i++ {
for j := i + 1; j < len(stations); j++ {
if stations[i].Code == stations[j].Code {
t.Errorf("station duplication: " + stations[i].Code)
}
}
}
}
func TestMarks(t *testing.T) {
var marks meta.MarkList
t.Log("Load network marks file")
{
if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
t.Fatal(err)
}
}
for i := 0; i < len(marks); i++ {
for j := i + 1; j < len(marks); j++ {
if marks[i].Code == marks[j].Code {
t.Errorf("mark duplication: " + marks[i].Code)
}
}
}
}
func TestRecorders(t *testing.T) {
var recorders meta.InstalledRecorderList
t.Log("Load installed recorders file")
{
if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil {
t.Fatal(err)
}
}
t.Log("Check for recorder installation equipment overlaps")
{
installs := make(map[string]meta.InstalledRecorderList)
for _, s := range recorders {
_, ok := installs[s.Model]
if ok {
installs[s.Model] = append(installs[s.Model], s)
} else {
installs[s.Model] = meta.InstalledRecorderList{s}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("recorder %s/%s at %-5s has location %-2s overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for missing recorder stations")
{
var stations meta.StationList
if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
t.Fatal(err)
}
keys := make(map[string]interface{})
for _, s := range stations {
keys[s.Code] = true
}
for _, s := range recorders {
if _, ok := keys[s.Station]; ok {
continue
}
t.Errorf("unable to find recorder installed station %-5s", s.Station)
}
}
var assets meta.AssetList
t.Log("Load recorder assets file")
{
if err := meta.LoadList("../assets/recorders.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for recorder assets")
{
for _, r := range recorders {
model := r.DataloggerModel
if r.DataloggerModel != r.Model {
model = strings.Join([]string{r.DataloggerModel, r.Model}, " ")
}
var found bool
for _, a := range assets {
if a.Model != model {
continue
}
if a.Serial != r.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find recorders asset: %s [%s]", model, r.Serial)
}
}
}
}
// TestStationsProto creates a Protobuf file of Stations.
func TestStationsProto(t *testing.T) {
var networks meta.NetworkList
if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
t.Error(err)
}
var net = make(map[string]*delta.Network)
for _, v := range networks {
n := delta.Network{
Code: v.Code,
External: v.External,
Description: v.Description,
Restricted: v.Restricted,
}
net[v.Code] = &n
}
var stations meta.StationList
if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
t.Error(err)
}
if len(stations) == 0 {
t.Error("zero length stations list.")
}
var s delta.Stations
s.Stations = make(map[string]*delta.Station)
for _, v := range stations {
pt := delta.Point{
Longitude: v.Longitude,
Latitude: v.Latitude,
Elevation: v.Elevation,
Datum: v.Datum,
}
sp := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
st := delta.Station{
Code: v.Code,
Name: v.Name,
Network: net[v.Network],
Point: &pt,
Span: &sp,
}
s.Stations[st.Code] = &st
}
b, err := proto.Marshal(&s)
if err != nil {
t.Error(err)
}
t.Log(s)
if err := os.MkdirAll(apiDir, 0777); err != nil {
t.Error(err)
}
if err := ioutil.WriteFile(apiDir+"/stations.pb", b, 0644); err != nil {
t.Error(err)
}
}
// TestMarksProto creates protobuf and JSON files of Marks.
// These are pushed to S3 (by Travis) for use in api.geonet.org.nz
// Three files are created:
// marks.pb - fully hydrated protobuf will all GNSS Mark information.
// marks.json - JSON version of marks.pb (for use in browsers).
// marks.geojson - GeoJSON of Mark locations.
func TestMarksProto(t *testing.T) {
var networks meta.NetworkList
if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
t.Error(err)
}
var net = make(map[string]*delta.Network)
for _, v := range networks {
n := delta.Network{
Code: v.Code,
External: v.External,
Description: v.Description,
Restricted: v.Restricted,
}
net[v.Code] = &n
}
var marks meta.MarkList
if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
t.Error(err)
}
if len(marks) == 0 {
t.Error("zero length mark list.")
}
var m delta.Marks
m.Marks = make(map[string]*delta.Mark)
for _, v := range marks {
pt := delta.Point{
Longitude: v.Longitude,
Latitude: v.Latitude,
Elevation: v.Elevation,
Datum: v.Datum,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
mk := delta.Mark{
Code: v.Code,
Name: v.Name,
Network: net[v.Network],
Point: &pt,
Span: &s,
}
m.Marks[mk.Code] = &mk
}
var monuments meta.MonumentList
if err := meta.LoadList("../network/monuments.csv", &monuments); err != nil {
t.Error(err)
}
for _, v := range monuments {
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
mn := delta.Monument{
DomesNumber: v.DomesNumber,
MarkType: v.MarkType,
Type: v.Type,
GroundRelationship: v.GroundRelationship,
FoundationType: v.FoundationType,
FoundationDepth: v.FoundationDepth,
Bedrock: v.Bedrock,
Geology: v.Geology,
Span: &s,
}
if _, ok := m.Marks[v.Mark]; ok {
m.Marks[v.Mark].Monument = &mn
}
}
var antennas meta.InstalledAntennaList
if err := meta.LoadList("../install/antennas.csv", &antennas); err != nil {
t.Error(err)
}
for _, v := range antennas {
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
}
o := delta.Offset{
Vertical: v.Vertical,
North: v.North,
East: v.East,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
a := delta.InstalledAntenna{
Equipment: &e,
Offset: &o,
Span: &s,
Azimuth: v.Azimuth,
}
if _, ok := m.Marks[v.Mark]; ok {
m.Marks[v.Mark].InstalledAntenna = append(m.Marks[v.Mark].InstalledAntenna, &a)
}
}
var radomes meta.InstalledRadomeList
if err := meta.LoadList("../install/radomes.csv", &radomes); err != nil {
t.Error(err)
}
for _, v := range radomes {
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
r := delta.InstalledRadome{
Equipment: &e,
Span: &s,
}
if _, ok := m.Marks[v.Mark]; ok {
m.Marks[v.Mark].InstalledRadome = append(m.Marks[v.Mark].InstalledRadome, &r)
}
}
var firmwares meta.FirmwareHistoryList
if err := meta.LoadList("../install/firmware.csv", &firmwares); err != nil {
t.Error(err)
}
var fw = make(map[delta.Equipment]delta.Receiver)
for _, v := range firmwares {
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
f := delta.Firmware{
Version: v.Version,
Notes: v.Notes,
Span: &s,
}
rx := fw[e]
rx.Equipment = &e
rx.Firmware = append(rx.Firmware, &f)
fw[e] = rx
}
var receivers meta.DeployedReceiverList
if err := meta.LoadList("../install/receivers.csv", &receivers); err != nil {
t.Error(err)
}
for _, v := range receivers {
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
rx, ok := fw[e]
if !ok {
t.Errorf("no firware for %v", e)
}
d := delta.DeployedReceiver{
Receiver: &rx,
Span: &s,
}
if _, ok := m.Marks[v.Mark]; ok {
m.Marks[v.Mark].DeployedReceiver = append(m.Marks[v.Mark].DeployedReceiver, &d)
}
}
var sessions meta.SessionList
if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil {
t.Error(err)
}
for _, v := range sessions {
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
se := delta.Session{
Operator: v.Operator,
Agency: v.Agency,
Model: v.Model,
SatelliteSystem: v.SatelliteSystem,
Interval: v.Interval.Nanoseconds(),
ElevationMask: v.ElevationMask,
HeaderComment: v.HeaderComment,
Span: &s,
}
if _, ok := m.Marks[v.Mark]; ok {
m.Marks[v.Mark].Session = append(m.Marks[v.Mark].Session, &se)
}
}
var metsensors meta.InstalledMetSensorList
if err := meta.LoadList("../install/metsensors.csv", &metsensors); err != nil {
t.Error(err)
}
for _, v := range metsensors {
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
p := delta.Point{
Longitude: v.Longitude,
Latitude: v.Latitude,
Elevation: v.Elevation,
Datum: v.Datum,
}
ms := delta.InstalledMetSensor{
Equipment: &e,
Span: &s,
Point: &p,
IMSComment: v.IMSComment,
}
if _, ok := m.Marks[v.Mark]; ok {
m.Marks[v.Mark].InstalledMetSensor = append(m.Marks[v.Mark].InstalledMetSensor, &ms)
}
}
// output files
if err := os.MkdirAll(apiDir, 0777); err != nil {
t.Error(err)
}
b, err := proto.Marshal(&m)
if err != nil {
t.Error(err)
}
if err := ioutil.WriteFile(apiDir+"/marks.pb", b, 0644); err != nil {
t.Error(err)
}
b, err = json.Marshal(&m)
if err != nil {
t.Error(err)
}
if err := ioutil.WriteFile(apiDir+"/marks.json", b, 0644); err != nil {
t.Error(err)
}
// GeoJSON files of site for each sensor type.
// This is similar to the sensor type output in stations_test.go
// There is no other sensor type lookup so handle that switching here.
// There is fractionally more work as this is two different installed types
// (antenna and metsensor).
out := map[string]*bytes.Buffer{
"metsensor": &bytes.Buffer{},
"gpsantenna": &bytes.Buffer{},
}
for k := range out {
out[k].WriteString(`{"type": "FeatureCollection","features": [`)
}
for _, mark := range m.Marks {
if mark.GetSpan() == nil || mark.GetPoint() == nil || mark.GetNetwork() == nil {
continue
}
if mark.InstalledAntenna != nil && len(mark.InstalledAntenna) > 0 {
for _, v := range mark.InstalledAntenna {
writeAntennaProps(mark, v, out["gpsantenna"])
}
}
if mark.InstalledMetSensor != nil && len(mark.InstalledMetSensor) > 0 {
for _, v := range mark.InstalledMetSensor {
writeMetsensorProps(mark, v, out["metsensor"])
}
}
}
for k := range out {
out[k].WriteString(`]}`)
fn := apiDir + "/" + k + ".geojson"
if err := ioutil.WriteFile(fn, out[k].Bytes(), 0644); err != nil {
t.Error(err)
}
}
}
func TestSensors(t *testing.T) {
var installed meta.InstalledSensorList
t.Log("Load installed sensors file")
{
if err := meta.LoadList("../install/sensors.csv", &installed); err != nil {
t.Fatal(err)
}
sort.Sort(installed)
}
t.Log("Check for missing sensors")
{
var sensors meta.AssetList
if err := meta.LoadList("../assets/sensors.csv", &sensors); err != nil {
t.Fatal(err)
}
sort.Sort(sensors)
for _, i := range installed {
n := sort.Search(len(sensors), func(j int) bool { return !sensors[j].Equipment.Less(i.Equipment) })
if n < 0 || i.Equipment.Less(sensors[n].Equipment) {
t.Errorf("unable to find sensor: %s", i.String())
}
}
}
t.Log("Check for sensor installation overlaps")
{
installs := make(map[string]meta.InstalledSensorList)
for _, s := range installed {
_, ok := installs[s.Model]
if ok {
installs[s.Model] = append(installs[s.Model], s)
} else {
installs[s.Model] = meta.InstalledSensorList{s}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("sensor %s/%s at %-5s has location %-2s overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
/*
t.Log("Check for missing sensor stations")
{
var stations meta.StationList
if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
t.Fatal(err)
}
sort.Sort(stations)
for _, i := range installed {
n := sort.Search(len(stations), func(j int) bool { return !(stations[j].Code < i.StationCode) })
if n < 0 || i.StationCode != stations[n].Code {
t.Errorf("unable to find station: %s", i.StationCode)
} else {
if i.Start.Before(stations[n].Start) {
t.Errorf("installed sensor before station has been opened: %s: %s (%s %s)", i.String(), i.Start.String(), stations[n].Code, stations[n].Start.String())
}
if i.End.After(stations[n].End) {
t.Errorf("installed sensor after station has been closed: %s: %s (%s %s)", i.String(), i.End.String(), stations[n].Code, stations[n].End.String())
}
}
}
}
t.Log("Check for missing sensor sites")
{
var sites meta.SiteList
if err := meta.LoadList("../network/sites.csv", &sites); err != nil {
t.Fatal(err)
}
sort.Sort(sites)
for _, i := range installed {
n := sort.Search(len(sites), func(j int) bool {
if sites[j].StationCode > i.StationCode {
return true
}
if sites[j].StationCode < i.StationCode {
return false
}
return !(sites[j].LocationCode < i.LocationCode)
})
if n < 0 || i.StationCode != sites[n].StationCode || i.LocationCode != sites[n].LocationCode {
t.Errorf("unable to find site: %s/%s", i.StationCode, i.LocationCode)
} else {
if i.Start.Before(sites[n].Start) {
t.Errorf("installed sensor before site has been opened: %s: %s (%s/%s %s)", i.String(), i.Start.String(), sites[n].StationCode, sites[n].LocationCode, sites[n].Start.String())
}
if i.End.After(sites[n].End) {
t.Errorf("installed sensor after site has been closed: %s: %s (%s/%s %s)", i.String(), i.End.String(), sites[n].StationCode, sites[n].LocationCode, sites[n].End.String())
}
}
}
}
*/
for _, i := range installed {
if i.Orientation.Azimuth < -360.0 || i.Orientation.Azimuth > 360.0 {
t.Errorf("installed sensor has invalid orientation azimuth: %s [%g]", i.String(), i.Orientation.Azimuth)
}
if i.Orientation.Dip < -90.0 || i.Orientation.Dip > 90.0 {
t.Errorf("installed sensor has invalid orientation dip: %s [%g]", i.String(), i.Orientation.Dip)
}
}
var assets meta.AssetList
t.Log("Load receiver sensors file")
{
if err := meta.LoadList("../assets/sensors.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for sensor assets")
{
for _, s := range installed {
var found bool
for _, a := range assets {
if a.Model != s.Model {
continue
}
if a.Serial != s.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find sensor asset: %s [%s]", s.Model, s.Serial)
}
}
}
}
func TestReceivers(t *testing.T) {
var receivers meta.DeployedReceiverList
t.Log("Load deployed receivers file")
{
if err := meta.LoadList("../install/receivers.csv", &receivers); err != nil {
t.Fatal(err)
}
}
t.Log("Check for particular receiver installation overlaps")
{
installs := make(map[string]meta.DeployedReceiverList)
for _, s := range receivers {
if _, ok := installs[s.Model]; !ok {
installs[s.Model] = meta.DeployedReceiverList{}
}
installs[s.Model] = append(installs[s.Model], s)
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("receiver %s [%s] at %s has overlap with %s between times %s and %s",
v[i].Model, v[i].Serial, v[i].Mark, v[j].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for receiver sites installation equipment overlaps")
{
installs := make(map[string]meta.DeployedReceiverList)
for _, s := range receivers {
if _, ok := installs[s.Mark]; !ok {
installs[s.Mark] = meta.DeployedReceiverList{}
}
installs[s.Mark] = append(installs[s.Model], s)
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("receivers %s [%s] / %s [%s] at %s has overlap between %s and %s",
v[i].Model, v[i].Serial, v[j].Model, v[j].Serial, v[i].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for missing receiver marks")
{
var marks meta.MarkList
if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
t.Fatal(err)
}
keys := make(map[string]interface{})
for _, m := range marks {
keys[m.Code] = true
}
for _, r := range receivers {
if _, ok := keys[r.Mark]; ok {
continue
}
t.Errorf("unable to find receiver mark %-5s", r.Mark)
}
}
var assets meta.AssetList
t.Log("Load receiver assets file")
{
if err := meta.LoadList("../assets/receivers.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for receiver assets")
{
for _, r := range receivers {
var found bool
for _, a := range assets {
if a.Model != r.Model {
continue
}
if a.Serial != r.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find receiver asset: %s [%s]", r.Model, r.Serial)
}
}
}
}
func TestConnections(t *testing.T) {
var connections meta.ConnectionList
t.Log("Load connections file")
if err := meta.LoadList("../install/connections.csv", &connections); err != nil {
t.Fatal(err)
}
for i := 0; i < len(connections); i++ {
for j := i + 1; j < len(connections); j++ {
if connections[i].Station != connections[j].Station {
continue
}
if connections[i].Location != connections[j].Location {
continue
}
if connections[i].Start.After(connections[j].End) {
continue
}
if connections[i].End.Before(connections[j].Start) {
continue
}
t.Errorf("connection overlap: " + strings.Join([]string{
connections[i].Station,
connections[i].Location,
connections[i].Start.String(),
connections[i].End.String(),
}, " "))
}
}
stas := make(map[string]meta.Station)
{
var list meta.StationList
t.Log("Load stations file")
if err := meta.LoadList("../network/stations.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
stas[s.Code] = s
}
}
sites := make(map[string]map[string]meta.Site)
{
var list meta.SiteList
t.Log("Load sites file")
if err := meta.LoadList("../network/sites.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
if _, ok := sites[s.Station]; !ok {
sites[s.Station] = make(map[string]meta.Site)
}
sites[s.Station][s.Location] = s
}
}
for _, c := range connections {
if _, ok := stas[c.Station]; !ok {
t.Log("unknown connection station: " + c.Station)
} else if s, ok := sites[c.Station]; !ok {
t.Log("unknown connection station: " + c.Station)
} else if _, ok := s[c.Location]; !ok {
t.Log("unknown connection station/location: " + c.Station + "/" + c.Location)
}
if c.Start.After(c.End) {
t.Log("connection span mismatch: " + strings.Join([]string{
c.Station,
c.Location,
c.Start.String(),
"after",
c.End.String(),
}, " "))
}
}
places := make(map[string]string)
{
var list meta.DeployedDataloggerList
t.Log("Load installed dataloggers file")
if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil {
t.Fatal(err)
}
for _, d := range list {
if d.Role != "" {
places[d.Place+"/"+d.Role] = d.Place
} else {
places[d.Place] = d.Place
}
}
}
for _, c := range connections {
if c.Role != "" {
if _, ok := places[c.Place+"/"+c.Role]; !ok {
t.Log("warning: unknown datalogger place/role: " + c.Place + "/" + c.Role)
}
} else {
if _, ok := places[c.Place]; !ok {
t.Log("warning: unknown datalogger place: " + c.Place)
}
}
}
var missing []meta.Connection
var assets = make(map[string]meta.Asset)
{
var list meta.AssetList
t.Log("Load datalogger assets file")
if err := meta.LoadList("../assets/dataloggers.csv", &list); err != nil {
t.Fatal(err)
}
for _, d := range list {
assets[d.Model+":::"+d.Serial] = d
}
t.Log("Load sensor assets file")
if err := meta.LoadList("../assets/sensors.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
assets[s.Model+":::"+s.Serial] = s
}
}
var sensors []meta.InstalledSensor
{
var list meta.InstalledSensorList
t.Log("Load stations file")
if err := meta.LoadList("../install/sensors.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
sensors = append(sensors, s)
}
}
for _, s := range sensors {
if a, ok := assets[s.Model+":::"+s.Serial]; !ok || a.Number == "" {
continue
}
if s.End.Before(time.Now()) {
continue
}
var handled bool
for _, c := range connections {
if c.Station != s.Station || c.Location != s.Location {
continue
}
if c.Start.After(s.End) || c.End.Before(s.Start) {
continue
}
handled = true
}
if !handled {
var place string
if p, ok := stas[s.Station]; ok {
place = p.Name
}
missing = append(missing, meta.Connection{
Station: s.Station,
Location: s.Location,
Place: place,
Span: meta.Span{
Start: s.Start,
End: s.End,
},
})
t.Errorf("no current connection defined for sensor: %s [%s/%s] %s %s", s.String(), s.Station, s.Location, s.Start, s.End)
}
}
var dataloggers []meta.DeployedDatalogger
{
var list meta.DeployedDataloggerList
t.Log("Load stations file")
if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil {
t.Fatal(err)
}
for _, d := range list {
dataloggers = append(dataloggers, d)
}
}
for _, d := range dataloggers {
if a, ok := assets[d.Model+":::"+d.Serial]; !ok || a.Number == "" {
continue
}
if d.End.Before(time.Now()) {
continue
}
var handled bool
for _, c := range connections {
if c.Place != d.Place || c.Role != d.Role {
continue
}
if c.Start.After(d.End) || c.End.Before(d.Start) {
continue
}
handled = true
}
if !handled {
s, l := "XXXX", "LL"
for k, v := range stas {
if v.Name == d.Place {
s = k
}
}
missing = append(missing, meta.Connection{
Station: s,
Location: l,
Place: d.Place,
Role: d.Role,
Span: meta.Span{
Start: d.Start,
End: d.End,
},
})
t.Errorf("no current connection defined for datalogger: %s [%s/%s] %s %s", d.String(), d.Place, d.Role, d.Start, d.End)
}
}
if len(missing) > 0 {
sort.Sort(meta.ConnectionList(missing))
t.Log("\n" + string(meta.MarshalList(meta.ConnectionList(missing))))
}
}
func TestStreams(t *testing.T) {
var streams meta.StreamList
t.Log("Load streams file")
if err := meta.LoadList("../install/streams.csv", &streams); err != nil {
t.Fatal(err)
}
for i := 0; i < len(streams); i++ {
for j := i + 1; j < len(streams); j++ {
if streams[i].Station != streams[j].Station {
continue
}
if streams[i].Location != streams[j].Location {
continue
}
if streams[i].Start.After(streams[j].End) {
continue
}
if streams[i].End.Before(streams[j].Start) {
continue
}
if streams[i].SamplingRate != streams[j].SamplingRate {
continue
}
t.Errorf("stream overlap: " + strings.Join([]string{
streams[i].Station,
streams[i].Location,
streams[i].Start.String(),
streams[i].End.String(),
}, " "))
}
}
stas := make(map[string]meta.Station)
{
var list meta.StationList
t.Log("Load stations file")
if err := meta.LoadList("../network/stations.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
stas[s.Code] = s
}
}
sites := make(map[string]map[string]meta.Site)
{
var list meta.SiteList
t.Log("Load sites file")
if err := meta.LoadList("../network/sites.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
if _, ok := sites[s.Station]; !ok {
sites[s.Station] = make(map[string]meta.Site)
}
sites[s.Station][s.Location] = s
}
}
for _, c := range streams {
if _, ok := stas[c.Station]; !ok {
t.Log("unknown stream station: " + c.Station)
} else if s, ok := sites[c.Station]; !ok {
t.Log("unknown stream station: " + c.Station)
} else if _, ok := s[c.Location]; !ok {
t.Log("unknown stream station/location: " + c.Station + "/" + c.Location)
}
if c.Start.After(c.End) {
t.Log("stream span mismatch: " + strings.Join([]string{
c.Station,
c.Location,
c.Start.String(),
"after",
c.End.String(),
}, " "))
}
}
}
func TestAntennas(t *testing.T) {
var antennas meta.InstalledAntennaList
t.Log("Load deployed antennas file")
{
if err := meta.LoadList("../install/antennas.csv", &antennas); err != nil {
t.Fatal(err)
}
}
t.Log("Check for antenna installation equipment overlaps")
{
installs := make(map[string]meta.InstalledAntennaList)
for _, s := range antennas {
if _, ok := installs[s.Model]; !ok {
installs[s.Model] = meta.InstalledAntennaList{}
}
installs[s.Model] = append(installs[s.Model], s)
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("antennas %s [%s] at %s has overlap at %s between %s and %s",
v[i].Model, v[i].Serial, v[i].Mark, v[j].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for antenna installation mark overlaps")
{
installs := make(map[string]meta.InstalledAntennaList)
for _, s := range antennas {
if _, ok := installs[s.Mark]; !ok {
installs[s.Mark] = meta.InstalledAntennaList{}
}
installs[s.Mark] = append(installs[s.Mark], s)
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("antennas %s [%s] and %s [%s] at %s has overlap between %s and %s",
v[i].Model, v[i].Serial, v[j].Model, v[j].Serial, v[i].Mark,
v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for missing antenna marks")
{
var marks meta.MarkList
if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
t.Fatal(err)
}
keys := make(map[string]interface{})
for _, m := range marks {
keys[m.Code] = true
}
for _, c := range antennas {
if _, ok := keys[c.Mark]; ok {
continue
}
t.Errorf("unable to find antenna mark %-5s", c.Mark)
}
}
var assets meta.AssetList
t.Log("Load antenna assets file")
{
if err := meta.LoadList("../assets/antennas.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for antenna assets")
{
for _, r := range antennas {
var found bool
for _, a := range assets {
if a.Model != r.Model {
continue
}
if a.Serial != r.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find antenna asset: %s [%s]", r.Model, r.Serial)
}
}
}
var sessions meta.SessionList
t.Log("Load session list")
{
if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil {
t.Fatal(err)
}
}
t.Log("Check sessions exist ...")
{
for _, r := range antennas {
var found bool
for _, s := range sessions {
if s.End.Before(r.Start) {
continue
}
if s.Start.After(r.End) {
continue
}
found = true
}
if !found {
t.Log(r)
}
}
}
}
func TestFirmware(t *testing.T) {
var firmwares meta.FirmwareHistoryList
t.Log("Load firmware history file")
{
if err := meta.LoadList("../install/firmware.csv", &firmwares); err != nil {
t.Fatal(err)
}
}
t.Log("Check for firmware history overlaps")
{
installs := make(map[string]meta.FirmwareHistoryList)
for _, s := range firmwares {
_, ok := installs[s.Model]
if ok {
installs[s.Model] = append(installs[s.Model], s)
} else {
installs[s.Model] = meta.FirmwareHistoryList{s}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("firmware %s / %s has overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
var assets meta.AssetList
t.Log("Load firmware assets file")
{
if err := meta.LoadList("../assets/receivers.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for firmware receiver assets")
{
for _, r := range firmwares {
var found bool
for _, a := range assets {
if a.Model != r.Model {
continue
}
if a.Serial != r.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find firmware receiver asset: %s [%s]", r.Model, r.Serial)
}
}
}
}
func main() {
var verbose bool
flag.BoolVar(&verbose, "verbose", false, "make noise")
var output string
flag.StringVar(&output, "output", "output", "output directory")
var logs string
flag.StringVar(&logs, "logs", "logs", "logs output directory")
var network string
flag.StringVar(&network, "network", "../../network", "base network directory")
var install string
flag.StringVar(&install, "install", "../../install", "base install directory")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Build GNSS SiteLog XML files from delta meta information\n")
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Usage:\n")
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, " %s [options]\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Options:\n")
fmt.Fprintf(os.Stderr, "\n")
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, "\n")
}
flag.Parse()
var tplFuncMap template.FuncMap = template.FuncMap{
"empty": func(d, s string) string {
if s != "" {
return s
}
return d
},
"tolower": func(s string) string {
switch t := strings.ToLower(s); t {
case "wyatt/agnew drilled-braced":
return "Deep Wyatt/Agnew drilled-braced"
default:
return t
}
},
"lines": func(p, s string) string {
switch s {
case "":
return s
default:
return strings.Join(strings.Split(s, "\n"), "\n"+p)
}
},
"plus": func(n int) string {
return fmt.Sprintf("%-2s", strconv.Itoa(n+1))
},
"lat": func(s string) string {
if f, err := strconv.ParseFloat(s, 64); err == nil {
m := math.Abs(f-float64(int(f))) * 60.0
return fmt.Sprintf("%+3d%02d%05.2f", int(f), int(m), (m-float64(int(m)))*60.0)
}
return ""
},
"lon": func(s string) string {
if f, err := strconv.ParseFloat(s, 64); err == nil {
m := math.Abs(f-float64(int(f))) * 60.0
return fmt.Sprintf("%+3d%02d%05.2f", int(f), int(m), (m-float64(int(m)))*60.0)
}
return ""
},
}
tmpl, err := template.New("").Funcs(tplFuncMap).Parse(sitelogTemplate)
if err != nil {
log.Fatalf("error: unable to compile template: %v", err)
}
var firmwareHistoryList meta.FirmwareHistoryList
if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil {
log.Fatalf("error: unable to load firmware history: %v", err)
}
firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory)
for _, i := range firmwareHistoryList {
if _, ok := firmwareHistory[i.Model]; !ok {
firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory)
}
firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i)
}
for j, _ := range firmwareHistory {
for k, _ := range firmwareHistory[j] {
sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k]))
}
}
var installedAntennaList meta.InstalledAntennaList
if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil {
log.Fatalf("error: unable to load antenna installs: %v", err)
}
installedAntenna := make(map[string][]meta.InstalledAntenna)
for _, i := range installedAntennaList {
installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i)
}
for i, _ := range installedAntenna {
sort.Sort(meta.InstalledAntennaList(installedAntenna[i]))
}
var deployedReceiverList meta.DeployedReceiverList
if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil {
log.Fatalf("error: unable to load receiver installs: %v", err)
}
deployedReceivers := make(map[string][]meta.DeployedReceiver)
for _, i := range deployedReceiverList {
deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i)
}
for i, _ := range deployedReceivers {
sort.Sort(meta.DeployedReceiverList(deployedReceivers[i]))
}
var installedRadomeList meta.InstalledRadomeList
if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil {
log.Fatalf("error: unable to load radome installs: %v", err)
}
installedRadomes := make(map[string][]meta.InstalledRadome)
for _, i := range installedRadomeList {
installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i)
}
for i, _ := range installedRadomes {
sort.Sort(meta.InstalledRadomeList(installedRadomes[i]))
}
var installedMetSensorList meta.InstalledMetSensorList
if err := meta.LoadList(filepath.Join(install, "metsensors.csv"), &installedMetSensorList); err != nil {
log.Fatalf("error: unable to load metsensors list: %v", err)
}
installedMetSensors := make(map[string][]meta.InstalledMetSensor)
for _, i := range installedMetSensorList {
installedMetSensors[i.Mark] = append(installedMetSensors[i.Mark], i)
}
for i, _ := range installedMetSensors {
sort.Sort(meta.InstalledMetSensorList(installedMetSensors[i]))
}
var markList meta.MarkList
if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil {
log.Fatalf("error: unable to load mark list: %v", err)
}
var sessionList meta.SessionList
if err := meta.LoadList(filepath.Join(install, "sessions.csv"), &sessionList); err != nil {
log.Fatalf("error: unable to load session list: %v", err)
}
sessions := make(map[string][]meta.Session)
for _, s := range sessionList {
sessions[s.Mark] = append(sessions[s.Mark], s)
}
var monumentList meta.MonumentList
if err := meta.LoadList(filepath.Join(network, "monuments.csv"), &monumentList); err != nil {
log.Fatalf("error: unable to load monument list: %v", err)
}
monuments := make(map[string]meta.Monument)
for _, m := range monumentList {
monuments[m.Mark] = m
}
for _, m := range markList {
if _, ok := monuments[m.Code]; !ok {
continue
}
if _, ok := sessions[m.Code]; !ok {
continue
}
if _, ok := installedAntenna[m.Code]; !ok {
continue
}
if _, ok := deployedReceivers[m.Code]; !ok {
continue
}
var receivers []GnssReceiver
var antennas []GnssAntenna
var metsensors []GnssMetSensor
for _, m := range installedMetSensors[m.Reference.Code] {
var session *meta.Session
for i, s := range sessions[m.Mark] {
if m.Start.After(s.End) || m.End.Before(s.Start) {
continue
}
session = &sessions[m.Mark][i]
break
}
if session == nil {
continue
}
metsensors = append(metsensors, GnssMetSensor{
Manufacturer: m.Make,
MetSensorModel: m.Model,
SerialNumber: m.Serial,
DataSamplingInterval: "360 sec",
EffectiveDates: "2000-02-05/CCYY-MM-DD",
Notes: "",
})
}
for _, a := range installedAntenna[m.Code] {
var session *meta.Session
for i, s := range sessions[m.Code] {
if a.Start.After(s.End) || a.End.Before(s.Start) {
continue
}
session = &sessions[m.Code][i]
break
}
if session == nil {
continue
}
radome := "NONE"
serial := ""
if _, ok := installedRadomes[m.Code]; ok {
for _, v := range installedRadomes[m.Code] {
if v.Start.After(a.End) || v.End.Before(a.Start) {
continue
}
radome = v.Model
serial = v.Serial
}
}
antennas = append(antennas, GnssAntenna{
AntennaType: a.Model,
SerialNumber: a.Serial,
AntennaReferencePoint: "BAM",
MarkerArpUpEcc: strconv.FormatFloat(a.Vertical, 'f', 4, 64),
MarkerArpNorthEcc: strconv.FormatFloat(a.North, 'f', 4, 64),
MarkerArpEastEcc: strconv.FormatFloat(a.East, 'f', 4, 64),
AlignmentFromTrueNorth: "0",
AntennaRadomeType: radome,
RadomeSerialNumber: serial,
AntennaCableType: "",
AntennaCableLength: "",
DateInstalled: a.Start.Format(DateTimeFormat),
DateRemoved: func() string {
if time.Now().After(a.End) {
return a.End.Format(DateTimeFormat)
} else {
return ""
}
}(),
Notes: "",
})
}
for _, r := range deployedReceivers[m.Code] {
if _, ok := firmwareHistory[r.Model]; ok {
if _, ok := firmwareHistory[r.Model][r.Serial]; ok {
for i, _ := range firmwareHistory[r.Model][r.Serial] {
v := firmwareHistory[r.Model][r.Serial][len(firmwareHistory[r.Model][r.Serial])-i-1]
if v.End.Before(r.Start) || v.Start.After(r.End) {
continue
}
var session *meta.Session
for i, s := range sessions[m.Code] {
if r.Start.After(s.End) || r.End.Before(s.Start) {
continue
}
if v.Start.After(s.End) || v.End.Before(s.Start) {
continue
}
session = &sessions[m.Code][i]
break
}
if session == nil {
continue
}
start := r.Start
/*
if start.Before(s.Start) {
start = s.Start
}
*/
if start.Before(v.Start) {
start = v.Start
}
end := r.End
/*
if end.After(s.End) {
end = s.End
}
*/
if end.After(v.End) {
end = v.End
}
receivers = append(receivers, GnssReceiver{
ReceiverType: r.Model,
SatelliteSystem: session.SatelliteSystem,
SerialNumber: r.Serial,
FirmwareVersion: v.Version,
ElevationCutoffSetting: strconv.FormatFloat(session.ElevationMask, 'g', -1, 64),
DateInstalled: start.Format(DateTimeFormat),
/*
DateInstalled: func() string {
if v.Start.Before(r.Start) {
return r.Start.Format(DateTimeFormat)
} else {
return v.Start.Format(DateTimeFormat)
}
}(),
*/
DateRemoved: func() string {
/*
if v.End.After(r.End) {
if time.Now().After(r.End) {
return r.End.Format(DateTimeFormat)
} else {
return ""
}
} else {
if time.Now().After(v.End) {
return v.End.Format(DateTimeFormat)
} else {
return ""
}
}
*/
if time.Now().After(end) {
return end.Format(DateTimeFormat)
} else {
return ""
}
}(),
TemperatureStabilization: "",
Notes: "",
})
}
}
}
}
sort.Sort(GnssReceivers(receivers))
sort.Sort(GnssAntennas(antennas))
monument := monuments[m.Code]
X, Y, Z := WGS842ITRF(m.Latitude, m.Longitude, m.Elevation)
x := SiteLog{
EquipNameSpace: equipNameSpace,
ContactNameSpace: contactNameSpace,
MiNameSpace: miNameSpace,
LiNameSpace: liNameSpace,
XmlNameSpace: xmlNameSpace,
XsiNameSpace: xsiNameSpace,
SchemaLocation: schemaLocation,
FormInformation: FormInformation{
PreparedBy: preparedBy,
DatePrepared: time.Now().Format(DateFormat),
ReportType: "DYNAMIC",
},
SiteIdentification: SiteIdentification{
SiteName: m.Name,
FourCharacterID: m.Code,
MonumentInscription: "",
IersDOMESNumber: monument.DomesNumber,
CdpNumber: "",
MonumentDescription: monument.Type,
HeightOfTheMonument: strconv.FormatFloat(-monument.GroundRelationship, 'g', -1, 64),
MonumentFoundation: monument.FoundationType,
FoundationDepth: strconv.FormatFloat(monument.FoundationDepth, 'f', 1, 64),
MarkerDescription: func() string {
switch monument.MarkType {
case "Forced Centering":
return "Forced Centering"
default:
return "unknown"
}
}(),
DateInstalled: m.Start.Format(DateTimeFormat),
GeologicCharacteristic: "",
BedrockType: "",
BedrockCondition: "",
FractureSpacing: "",
FaultZonesNearby: "",
DistanceActivity: "",
Notes: "",
},
SiteLocation: SiteLocation{
/*
City: m.Place,
State: m.Region,
*/
Country: func(lat, lon float64) string {
X, Y, _ := WGS842ITRF(lat, lon, 0.0)
dist := float64(-1.0)
country := "Unknown"
for _, v := range countryList {
x, y, _ := WGS842ITRF(v.lat, v.lon, 0.0)
r := math.Sqrt((x-X)*(x-X) + (y-Y)*(y-Y))
if dist < 0.0 || r < dist {
country = v.name
dist = r
}
}
return country
}(m.Latitude, m.Longitude),
TectonicPlate: TectonicPlate(m.Latitude, m.Longitude),
ApproximatePositionITRF: ApproximatePositionITRF{
XCoordinateInMeters: strconv.FormatFloat(X, 'f', 1, 64),
YCoordinateInMeters: strconv.FormatFloat(Y, 'f', 1, 64),
ZCoordinateInMeters: strconv.FormatFloat(Z, 'f', 1, 64),
LatitudeNorth: strconv.FormatFloat(m.Latitude, 'g', -1, 64),
LongitudeEast: strconv.FormatFloat(m.Longitude, 'g', -1, 64),
ElevationMEllips: strconv.FormatFloat(m.Elevation, 'f', 1, 64),
},
Notes: "",
},
GnssReceivers: receivers,
GnssAntennas: antennas,
GnssMetSensors: metsensors,
ContactAgency: contactAgency,
ResponsibleAgency: func() Agency {
switch m.Network {
case "LI":
return responsibleAgency
default:
return Agency{
MailingAddress: "\n",
}
}
}(),
MoreInformation: MoreInformation{
PrimaryDataCenter: primaryDatacentre,
SecondaryDataCenter: "",
UrlForMoreInformation: urlForMoreInformation,
HardCopyOnFile: "",
SiteMap: "",
SiteDiagram: "",
HorizonMask: "",
MonumentDescription: "",
SitePictures: "",
Notes: extraNotes + " " + m.Code,
AntennaGraphicsWithDimensions: func() string {
var graphs []string
models := make(map[string]interface{})
for _, a := range antennas {
if _, ok := models[a.AntennaType]; ok {
continue
}
if g, ok := antennaGraphs[a.AntennaType]; ok {
b, err := hex.DecodeString(g)
if err != nil {
log.Printf("error: unable to decode antenna graph for: \"%s\"", a.AntennaType)
continue
}
graphs = append(graphs, strings.Join([]string{a.AntennaType, string(b)}, "\n"))
} else {
log.Printf("warning: missing antenna graph for: \"%s\"", a.AntennaType)
}
models[a.AntennaType] = true
}
return strings.Join(graphs, "\n") + "\n"
}(),
InsertTextGraphicFromAntenna: "",
},
}
s, err := x.Marshal()
if err != nil {
log.Fatalf("error: unable to marshal xml: %v", err)
}
xmlfile := filepath.Join(output, strings.ToLower(m.Code)+".xml")
if err := os.MkdirAll(filepath.Dir(xmlfile), 0755); err != nil {
log.Fatalf("error: unable to create dir: %v", err)
}
if err := ioutil.WriteFile(xmlfile, s, 0644); err != nil {
log.Fatalf("error: unable to write file: %v", err)
}
logfile := filepath.Join(logs, strings.ToLower(m.Code)+".log")
if err := os.MkdirAll(filepath.Dir(logfile), 0755); err != nil {
log.Fatalf("error: unable to create logs dir: %v", err)
}
f, err := os.Create(logfile)
if err != nil {
log.Fatalf("error: unable to create log file: %v", err)
}
defer f.Close()
if err := tmpl.Execute(f, x); err != nil {
log.Fatalf("error: unable to write log file: %v", err)
}
}
}
func main() {
var verbose bool
flag.BoolVar(&verbose, "verbose", false, "make noise")
var output string
flag.StringVar(&output, "output", "output", "output directory")
var network string
flag.StringVar(&network, "network", "../../network", "base network directory")
var install string
flag.StringVar(&install, "install", "../../install", "base install directory")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Build a RINEX configuration site XML file from delta meta information\n")
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Usage:\n")
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, " %s [options]\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Options:\n")
fmt.Fprintf(os.Stderr, "\n")
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, "\n")
}
flag.Parse()
var firmwareHistoryList meta.FirmwareHistoryList
if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to load firmware history: %v\n", err)
os.Exit(-1)
}
firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory)
for _, i := range firmwareHistoryList {
if _, ok := firmwareHistory[i.Model]; !ok {
firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory)
}
firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i)
}
for j, _ := range firmwareHistory {
for k, _ := range firmwareHistory[j] {
sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k]))
}
}
var installedAntennaList meta.InstalledAntennaList
if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to load antenna installs: %v\n", err)
os.Exit(-1)
}
installedAntenna := make(map[string][]meta.InstalledAntenna)
for _, i := range installedAntennaList {
installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i)
}
for i, _ := range installedAntenna {
sort.Sort(meta.InstalledAntennaList(installedAntenna[i]))
}
var deployedReceiverList meta.DeployedReceiverList
if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to load receiver installs: %v\n", err)
os.Exit(-1)
}
deployedReceivers := make(map[string][]meta.DeployedReceiver)
for _, i := range deployedReceiverList {
deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i)
}
for i, _ := range deployedReceivers {
sort.Sort(meta.DeployedReceiverList(deployedReceivers[i]))
}
var installedRadomeList meta.InstalledRadomeList
if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to load radome installs: %v\n", err)
os.Exit(-1)
}
installedRadomes := make(map[string][]meta.InstalledRadome)
for _, i := range installedRadomeList {
installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i)
}
for i, _ := range installedRadomes {
sort.Sort(meta.InstalledRadomeList(installedRadomes[i]))
}
var markList meta.MarkList
if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to load mark list: %v\n", err)
os.Exit(-1)
}
var sessionList meta.SessionList
if err := meta.LoadList(filepath.Join(install, "sessions.csv"), &sessionList); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to load session list: %v\n", err)
os.Exit(-1)
}
sessions := make(map[string][]meta.Session)
for _, s := range sessionList {
sessions[s.Mark] = append(sessions[s.Mark], s)
}
for _, m := range markList {
if _, ok := sessions[m.Code]; !ok {
continue
}
if _, ok := installedAntenna[m.Code]; !ok {
continue
}
if _, ok := deployedReceivers[m.Code]; !ok {
continue
}
var list []CGPSSessionXML
for _, s := range sessions[m.Code] {
for _, a := range installedAntenna[m.Code] {
if a.Start.After(s.End) || a.End.Before(s.Start) {
continue
}
if _, ok := IGSModels[a.Model]; !ok {
fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s], skipping\n", m.Code, a.Model, a.Serial)
continue
}
for _, r := range deployedReceivers[m.Code] {
if r.Start.After(a.End) || r.End.Before(a.Start) {
continue
}
if _, ok := IGSModels[r.Model]; !ok {
fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s]", m.Code, r.Model, r.Serial)
continue
}
radome := "NONE"
if _, ok := installedRadomes[m.Code]; ok {
for _, v := range installedRadomes[m.Code] {
if v.Start.After(a.End) || v.Start.After(r.End) {
continue
}
if v.End.Before(a.Start) || v.End.Before(r.Start) {
continue
}
if _, ok := IGSModels[v.Model]; !ok {
fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s]", m.Code, v.Model, v.Serial)
continue
}
radome = IGSModels[v.Model]
}
}
var firmware []FirmwareHistoryXML
if _, ok := firmwareHistory[r.Model]; ok {
if _, ok := firmwareHistory[r.Model][r.Serial]; ok {
for i, _ := range firmwareHistory[r.Model][r.Serial] {
v := firmwareHistory[r.Model][r.Serial][len(firmwareHistory[r.Model][r.Serial])-i-1]
if v.End.Before(r.Start) || v.Start.After(r.End) {
continue
}
firmware = append(firmware, FirmwareHistoryXML{
StartTime: v.Start.Format(meta.DateTimeFormat),
StopTime: func() string {
if time.Now().After(v.End) {
return v.End.Format(meta.DateTimeFormat)
} else {
return "open"
}
}(),
Version: v.Version,
})
}
}
}
list = append(list, CGPSSessionXML{
StartTime: func() string {
if r.Start.After(a.Start) {
return r.Start.Format(meta.DateTimeFormat)
} else if a.Start.After(s.Start) {
return a.Start.Format(meta.DateTimeFormat)
} else {
return s.Start.Format(meta.DateTimeFormat)
}
}(),
StopTime: func() string {
if r.End.Before(a.End) {
if time.Now().After(r.End) {
return r.End.Format(meta.DateTimeFormat)
} else {
return "open"
}
} else if a.End.Before(s.End) {
if time.Now().After(a.End) {
return a.End.Format(meta.DateTimeFormat)
} else {
return "open"
}
} else {
if time.Now().After(s.End) {
return s.End.Format(meta.DateTimeFormat)
} else {
return "open"
}
}
}(),
Receiver: ReceiverXML{
SerialNumber: r.Serial,
IGSDesignation: IGSModels[r.Model],
FirmwareHistories: firmware,
},
InstalledCGPSAntenna: InstalledCGPSAntennaXML{
Height: Number{Units: "m", Value: a.Vertical},
OffsetNorth: Number{Units: "m", Value: a.North},
OffsetEast: Number{Units: "m", Value: a.East},
Radome: radome,
CGPSAntenna: CGPSAntennaXML{
SerialNumber: a.Serial,
IGSDesignation: IGSModels[a.Model],
},
},
ObservationInterval: Number{
Units: "s",
Value: s.Interval.Seconds(),
},
Operator: OperatorXML{
Name: s.Operator,
Agency: s.Agency,
},
Rinex: RinexXML{
HeaderCommentName: s.HeaderComment,
HeaderCommentText: func() string {
if t, ok := HeaderComments[s.HeaderComment]; ok {
return strings.Join(strings.Fields(t), " ")
}
return ""
}(),
},
DataFormat: func() string {
if f, ok := DataFormats[r.Model]; ok {
return f
}
return "unknown"
}(),
DownloadNameFormat: func() DownloadNameFormatXML {
if f, ok := DownloadNameFormats[r.Model]; ok {
return f
}
return DownloadNameFormatXML{}
}(),
})
}
}
}
x := NewSiteXML(
MarkXML{
GeodeticCode: m.Code,
//DomesNumber: m.DomesNumber,
},
LocationXML{
Latitude: m.Latitude,
Longitude: m.Longitude,
Height: m.Elevation,
Datum: m.Datum,
},
list,
)
s, err := x.Marshal()
if err != nil {
fmt.Fprintf(os.Stderr, "error: unable to marsh xml: %v\n", err)
os.Exit(-1)
}
xmlfile := filepath.Join(output, m.Code+".xml")
if err := os.MkdirAll(filepath.Dir(xmlfile), 0755); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to create dir: %v\n", err)
os.Exit(-1)
}
if err := ioutil.WriteFile(xmlfile, s, 0644); err != nil {
fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err)
os.Exit(-1)
}
}
}
func TestStreams(t *testing.T) {
var streams meta.StreamList
t.Log("Load streams file")
if err := meta.LoadList("../install/streams.csv", &streams); err != nil {
t.Fatal(err)
}
for i := 0; i < len(streams); i++ {
for j := i + 1; j < len(streams); j++ {
if streams[i].Station != streams[j].Station {
continue
}
if streams[i].Location != streams[j].Location {
continue
}
if streams[i].Start.After(streams[j].End) {
continue
}
if streams[i].End.Before(streams[j].Start) {
continue
}
if streams[i].SamplingRate != streams[j].SamplingRate {
continue
}
t.Errorf("stream overlap: " + strings.Join([]string{
streams[i].Station,
streams[i].Location,
streams[i].Start.String(),
streams[i].End.String(),
}, " "))
}
}
stas := make(map[string]meta.Station)
{
var list meta.StationList
t.Log("Load stations file")
if err := meta.LoadList("../network/stations.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
stas[s.Code] = s
}
}
sites := make(map[string]map[string]meta.Site)
{
var list meta.SiteList
t.Log("Load sites file")
if err := meta.LoadList("../network/sites.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
if _, ok := sites[s.Station]; !ok {
sites[s.Station] = make(map[string]meta.Site)
}
sites[s.Station][s.Location] = s
}
}
for _, s := range streams {
if s.SamplingRate == 0 {
t.Errorf("invalid stream sample rate: " + strings.Join([]string{
s.Station,
s.Location,
s.Start.String(),
s.End.String(),
}, " "))
}
}
for _, c := range streams {
if _, ok := stas[c.Station]; !ok {
t.Log("unknown stream station: " + c.Station)
} else if s, ok := sites[c.Station]; !ok {
t.Log("unknown stream station: " + c.Station)
} else if _, ok := s[c.Location]; !ok {
t.Log("unknown stream station/location: " + c.Station + "/" + c.Location)
}
if c.Start.After(c.End) {
t.Log("stream span mismatch: " + strings.Join([]string{
c.Station,
c.Location,
c.Start.String(),
"after",
c.End.String(),
}, " "))
}
}
var assets = make(map[string]meta.Asset)
{
var list meta.AssetList
t.Log("Load recorders assets file")
if err := meta.LoadList("../assets/recorders.csv", &list); err != nil {
t.Fatal(err)
}
for _, l := range list {
assets[l.Model+":::"+l.Serial] = l
}
t.Log("Load sensors assets file")
if err := meta.LoadList("../assets/sensors.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
assets[s.Model+":::"+s.Serial] = s
}
}
var recorders []meta.InstalledRecorder
{
var list meta.InstalledRecorderList
t.Log("Load recorders file")
if err := meta.LoadList("../install/recorders.csv", &list); err != nil {
t.Fatal(err)
}
for _, r := range list {
recorders = append(recorders, r)
}
}
var missing []meta.Stream
for _, r := range recorders {
if a, ok := assets[r.Model+":::"+r.Serial]; !ok || a.Number == "" {
continue
}
if r.End.Before(time.Now()) {
continue
}
var handled bool
for _, s := range streams {
if s.Station != r.Station || r.Location != s.Location {
continue
}
if r.Start.After(s.End) || r.End.Before(s.Start) {
continue
}
handled = true
}
if !handled {
for _, sps := range recorderSamplingRates {
missing = append(missing, meta.Stream{
Station: r.Station,
Location: r.Location,
SamplingRate: sps,
Span: meta.Span{
Start: r.Start,
End: r.End,
},
})
}
t.Errorf("no current stream defined for recorder: %s [%s/%s] %s %s", r.String(), r.Station, r.Location, r.Start, r.End)
}
}
var sensors []meta.InstalledSensor
{
var list meta.InstalledSensorList
t.Log("Load sensors file")
if err := meta.LoadList("../install/sensors.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
sensors = append(sensors, s)
}
}
for _, v := range sensors {
if a, ok := assets[v.Model+":::"+v.Serial]; !ok || a.Number == "" {
continue
}
if v.End.Before(time.Now()) {
continue
}
var handled bool
for _, s := range streams {
if s.Station != v.Station || v.Location != s.Location {
continue
}
if v.Start.After(s.End) || v.End.Before(s.Start) {
continue
}
handled = true
}
if !handled {
for _, sps := range sensorSamplingRates {
missing = append(missing, meta.Stream{
Station: v.Station,
Location: v.Location,
SamplingRate: sps,
Span: meta.Span{
Start: v.Start,
End: v.End,
},
})
}
t.Errorf("no current stream defined for sensor: %s [%s/%s] %s %s", v.String(), v.Station, v.Location, v.Start, v.End)
}
}
if len(missing) > 0 {
sort.Sort(meta.StreamList(missing))
t.Log("\n" + string(meta.MarshalList(meta.StreamList(missing))))
}
}
// TestMarksProto creates a Protobuf file of Marks.
func TestMarksProto(t *testing.T) {
var networks meta.NetworkList
if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
t.Error(err)
}
var net = make(map[string]*delta.Network)
for _, v := range networks {
n := delta.Network{
Code: v.Code,
External: v.External,
Description: v.Description,
Restricted: v.Restricted,
}
net[v.Code] = &n
}
var marks meta.MarkList
if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
t.Error(err)
}
if len(marks) == 0 {
t.Error("zero length mark list.")
}
var m delta.Marks
m.Marks = make(map[string]*delta.Mark)
for _, v := range marks {
pt := delta.Point{
Longitude: v.Longitude,
Latitude: v.Latitude,
Elevation: v.Elevation,
Datum: v.Datum,
}
s := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
mk := delta.Mark{
Code: v.Code,
Name: v.Name,
Network: net[v.Network],
Point: &pt,
Span: &s,
}
m.Marks[mk.Code] = &mk
}
b, err := proto.Marshal(&m)
if err != nil {
t.Error(err)
}
if err := os.MkdirAll(apiDir, 0777); err != nil {
t.Error(err)
}
if err := ioutil.WriteFile(apiDir+"/marks.pb", b, 0644); err != nil {
t.Error(err)
}
}
func TestSessions(t *testing.T) {
var sessions meta.SessionList
t.Log("Load sessions file")
if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil {
t.Fatal(err)
}
for i := 0; i < len(sessions); i++ {
for j := i + 1; j < len(sessions); j++ {
if sessions[i].Mark != sessions[j].Mark {
continue
}
if sessions[i].Model != sessions[j].Model {
continue
}
if sessions[i].Interval != sessions[j].Interval {
continue
}
if sessions[i].Start.After(sessions[j].End) {
continue
}
if !sessions[i].End.After(sessions[j].Start) {
continue
}
t.Errorf("session overlap: " + strings.Join([]string{
sessions[i].Mark,
sessions[i].Interval.String(),
sessions[i].Start.String(),
sessions[i].End.String(),
sessions[j].Interval.String(),
sessions[j].Start.String(),
sessions[j].End.String(),
}, " "))
}
}
marks := make(map[string]meta.Mark)
{
var list meta.MarkList
t.Log("Load marks file")
if err := meta.LoadList("../network/marks.csv", &list); err != nil {
t.Fatal(err)
}
for _, m := range list {
marks[m.Code] = m
}
}
for _, s := range sessions {
if _, ok := marks[s.Mark]; !ok {
t.Log("unknown session mark: " + s.Mark)
}
if s.Start.After(s.End) {
t.Log("session span mismatch: " + strings.Join([]string{
s.Mark,
s.Interval.String(),
s.Start.String(),
"after",
s.End.String(),
}, " "))
}
switch s.SatelliteSystem {
case "GPS":
case "GPS+GLO":
case "GPS+GLO+GAL+BDS+QZSS":
default:
t.Error("unknown satellite system: " + s.SatelliteSystem)
}
}
}
func TestCombined(t *testing.T) {
var combined meta.InstalledSensorList
t.Log("Load installed sensors file")
{
if err := meta.LoadList("../install/sensors.csv", &combined); err != nil {
t.Fatal(err)
}
}
var recorders meta.InstalledRecorderList
t.Log("Load installed recorders file")
{
if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil {
t.Fatal(err)
}
}
for _, r := range recorders {
combined = append(combined, meta.InstalledSensor{
Install: r.Install,
Station: r.Station,
Location: r.Location,
})
}
t.Log("Check for sensor/recorder installation location overlaps")
{
installs := make(map[string]meta.InstalledSensorList)
for _, s := range combined {
_, ok := installs[s.Station]
if ok {
installs[s.Station] = append(installs[s.Station], s)
} else {
installs[s.Station] = meta.InstalledSensorList{s}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Location != v[j].Location:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("sensor/recorder %s/%s at %-5s has location %-2s overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
}
// TestStationsProto creates:
// Protobuf file of Stations
// JSON file of Stations
// GeoJSON of Site for each sensor type.
func TestStationsProto(t *testing.T) {
var networks meta.NetworkList
if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
t.Error(err)
}
var net = make(map[string]*delta.Network)
for _, v := range networks {
n := delta.Network{
Code: v.Code,
External: v.External,
Description: v.Description,
Restricted: v.Restricted,
}
net[v.Code] = &n
}
var stations meta.StationList
if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
t.Error(err)
}
if len(stations) == 0 {
t.Error("zero length stations list.")
}
var s delta.Stations
s.Stations = make(map[string]*delta.Station)
for _, v := range stations {
pt := delta.Point{
Longitude: v.Longitude,
Latitude: v.Latitude,
Elevation: v.Elevation,
Datum: v.Datum,
}
sp := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
st := delta.Station{
Code: v.Code,
Name: v.Name,
Network: net[v.Network],
Point: &pt,
Span: &sp,
}
s.Stations[st.Code] = &st
}
var sites meta.SiteList
if err := meta.LoadList("../network/sites.csv", &sites); err != nil {
t.Error(err)
}
for _, v := range sites {
pt := delta.Point{
Longitude: v.Longitude,
Latitude: v.Latitude,
Elevation: v.Elevation,
Datum: v.Datum,
}
sp := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
si := delta.Site{
Location: v.Location,
Survey: v.Survey,
Span: &sp,
Point: &pt,
}
if _, ok := s.Stations[v.Station]; ok {
if s.Stations[v.Station].Sites == nil {
s.Stations[v.Station].Sites = make(map[string]*delta.Site)
}
s.Stations[v.Station].Sites[si.Location] = &si
}
}
var installed meta.InstalledSensorList
if err := meta.LoadList("../install/sensors.csv", &installed); err != nil {
t.Fatal(err)
}
for _, v := range installed {
st := resp.SensorModels[v.Model]
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
Type: st.Type,
}
sp := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
o := delta.Orientation{
Dip: v.Dip,
Azimuth: v.Azimuth,
}
off := delta.Offset{
North: v.North,
East: v.East,
Vertical: v.Vertical,
}
sc := delta.Scale{
Bias: v.Bias,
Factor: v.Factor,
}
is := delta.InstalledSensor{
Equipment: &e,
Span: &sp,
Orientation: &o,
Offset: &off,
Scale: &sc,
}
if _, ok := s.Stations[v.Station]; ok {
if _, ok := s.Stations[v.Station].Sites[v.Location]; ok {
s.Stations[v.Station].Sites[v.Location].InstalledSensor = append(s.Stations[v.Station].Sites[v.Location].InstalledSensor, &is)
}
}
}
// strong motion recorders that are a sensor and a datalogger as a package.
var recorders meta.InstalledRecorderList
if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil {
t.Fatal(err)
}
for _, v := range recorders {
st, ok := resp.SensorModels[v.InstalledSensor.Model]
if !ok {
// in resp/auto.go some models have " SENSOR" appended.
st, ok = resp.SensorModels[v.InstalledSensor.Model+" SENSOR"]
}
e := delta.Equipment{
Make: v.Make,
Model: v.Model,
Serial: v.Serial,
Type: st.Type,
}
sp := delta.Span{
Start: v.Start.Unix(),
End: v.End.Unix(),
}
o := delta.Orientation{
Dip: v.Dip,
Azimuth: v.Azimuth,
}
off := delta.Offset{
North: v.North,
East: v.East,
Vertical: v.Vertical,
}
sc := delta.Scale{
Bias: v.Bias,
Factor: v.Factor,
}
is := delta.InstalledSensor{
Equipment: &e,
Span: &sp,
Orientation: &o,
Offset: &off,
Scale: &sc,
}
if _, ok := s.Stations[v.Station]; ok {
if _, ok := s.Stations[v.Station].Sites[v.Location]; ok {
s.Stations[v.Station].Sites[v.Location].InstalledSensor = append(s.Stations[v.Station].Sites[v.Location].InstalledSensor, &is)
}
}
}
// protobuf of all station information
b, err := proto.Marshal(&s)
if err != nil {
t.Error(err)
}
if err := os.MkdirAll(apiDir, 0777); err != nil {
t.Error(err)
}
if err := ioutil.WriteFile(apiDir+"/stations.pb", b, 0644); err != nil {
t.Error(err)
}
// json of all station information
b, err = json.Marshal(&s)
if err != nil {
t.Error(err)
}
if err := os.MkdirAll(apiDir, 0777); err != nil {
t.Error(err)
}
if err := ioutil.WriteFile(apiDir+"/stations.json", b, 0644); err != nil {
t.Error(err)
}
// GeoJSON files of site for each sensor type.
var out = make(map[string]*bytes.Buffer)
for _, v := range resp.SensorModels {
if v.Type != "" {
out[v.Type] = &bytes.Buffer{}
}
}
for k := range out {
out[k].WriteString(`{"type": "FeatureCollection","features": [`)
}
for _, station := range s.Stations {
if station.GetSpan() == nil || station.GetPoint() == nil || station.GetNetwork() == nil || station.Network.Restricted {
continue
}
if station.Network.External != "NZ" || station.Network.Code == "XX" {
continue
}
for _, site := range station.Sites {
for _, v := range site.InstalledSensor {
_, ok := out[v.Equipment.Type]
switch {
case v.Equipment.Type == "":
t.Logf("%s.%s no sensor model for %s, site not classified.",
station.Code, site.Location, v.Equipment.Model)
case ok:
writeProps(station, site, v, out[v.Equipment.Type])
case !ok:
t.Logf("unclassified type %s", v.Equipment.Type)
}
}
}
}
for k := range out {
out[k].WriteString(`]}`)
fn := strings.Replace(strings.ToLower(k), " ", "", -1)
fn = apiDir + "/" + fn + ".geojson"
if err := ioutil.WriteFile(fn, out[k].Bytes(), 0644); err != nil {
t.Error(err)
}
}
}
func TestDataloggers(t *testing.T) {
var dataloggers meta.DeployedDataloggerList
t.Log("Load deployed dataloggers file")
{
if err := meta.LoadList("../install/dataloggers.csv", &dataloggers); err != nil {
t.Fatal(err)
}
}
t.Log("Check for datalogger installation place overlaps")
{
installs := make(map[string]meta.DeployedDataloggerList)
for _, d := range dataloggers {
_, ok := installs[d.Place]
if ok {
installs[d.Place] = append(installs[d.Place], d)
} else {
installs[d.Place] = meta.DeployedDataloggerList{d}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Place != v[j].Place:
case v[i].Role != v[j].Role:
//case v[i].Model != v[j].Model:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
//case v[i].End.Equal(v[j].Start):
//case v[i].Start.Equal(v[j].End):
default:
t.Errorf("datalogger %s:[%s] at %-32s has place overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Place, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for datalogger installation equipment overlaps")
{
installs := make(map[string]meta.DeployedDataloggerList)
for _, s := range dataloggers {
_, ok := installs[s.Model]
if ok {
installs[s.Model] = append(installs[s.Model], s)
} else {
installs[s.Model] = meta.DeployedDataloggerList{s}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
// case v[i].End.Equal(v[j].Start):
// case v[i].Start.Equal(v[j].End):
default:
t.Errorf("datalogger %s:[%s] at %-32s has installation overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Place, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
var assets meta.AssetList
t.Log("Load datalogger assets file")
{
if err := meta.LoadList("../assets/dataloggers.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for datalogger assets")
{
for _, r := range dataloggers {
var found bool
for _, a := range assets {
if a.Model != r.Model {
continue
}
if a.Serial != r.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find datalogger asset: %s [%s]", r.Model, r.Serial)
}
}
}
}
func NewMetaDB(base string) (*MetaDB, error) {
db, err := memdb.NewMemDB(NewSchema())
if err != nil {
return nil, err
}
var networks = []struct {
table string
path string
list meta.List
}{
{"gauge", "network/gauges.csv", &meta.GaugeList{}},
{"constituent", "network/constituents.csv", &meta.ConstituentList{}},
{"mark", "network/marks.csv", &meta.MarkList{}},
{"monument", "network/monuments.csv", &meta.MonumentList{}},
{"mount", "network/mounts.csv", &meta.MountList{}},
{"network", "network/networks.csv", &meta.NetworkList{}},
{"site", "network/sites.csv", &meta.SiteList{}},
{"station", "network/stations.csv", &meta.StationList{}},
}
txn := db.Txn(true)
for _, list := range networks {
if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) {
continue
}
switch list.list.(type) {
case *meta.ConstituentList:
var input meta.ConstituentList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.GaugeList:
var input meta.GaugeList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.MarkList:
var input meta.MarkList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.MonumentList:
var input meta.MonumentList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.MountList:
var input meta.MountList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.NetworkList:
var input meta.NetworkList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.SiteList:
var input meta.SiteList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.StationList:
var input meta.StationList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
default:
return nil, fmt.Errorf("invalid type found for: %s", list.path)
}
}
txn.Commit()
var assets = []struct {
table string
path string
}{
{"antenna", "assets/antennas.csv"},
{"camera", "assets/cameras.csv"},
{"datalogger", "assets/dataloggers.csv"},
{"metsensor", "assets/metsensors.csv"},
{"radome", "assets/radomes.csv"},
{"receiver", "assets/receivers.csv"},
{"recorder", "assets/recorders.csv"},
{"sensor", "assets/sensors.csv"},
}
txn = db.Txn(true)
for _, list := range assets {
if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) {
continue
}
var input meta.AssetList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert("asset", i); err != nil {
return nil, err
}
}
}
txn.Commit()
var installs = []struct {
table string
path string
list meta.List
}{
{"antenna", "install/antennas.csv", &meta.InstalledAntennaList{}},
{"camera", "install/cameras.csv", &meta.InstalledCameraList{}},
{"datalogger", "install/dataloggers.csv", &meta.DeployedDataloggerList{}},
{"metsensor", "install/metsensors.csv", &meta.InstalledMetSensorList{}},
{"radome", "install/radomes.csv", &meta.InstalledRadomeList{}},
{"receiver", "install/receivers.csv", &meta.DeployedReceiverList{}},
{"recorder", "install/recorders.csv", &meta.InstalledRecorderList{}},
{"sensor", "install/sensors.csv", &meta.InstalledSensorList{}},
{"connection", "install/connections.csv", &meta.ConnectionList{}},
{"firmware", "install/firmware.csv", &meta.FirmwareHistoryList{}},
{"session", "install/sessions.csv", &meta.SessionList{}},
{"stream", "install/streams.csv", &meta.StreamList{}},
}
txn = db.Txn(true)
for _, list := range installs {
if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) {
continue
}
switch list.list.(type) {
case *meta.InstalledAntennaList:
var input meta.InstalledAntennaList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.InstalledCameraList:
var input meta.InstalledCameraList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.DeployedDataloggerList:
var input meta.DeployedDataloggerList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.InstalledMetSensorList:
var input meta.InstalledMetSensorList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.InstalledRadomeList:
var input meta.InstalledRadomeList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.DeployedReceiverList:
var input meta.DeployedReceiverList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.InstalledRecorderList:
var input meta.InstalledRecorderList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.InstalledSensorList:
var input meta.InstalledSensorList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.ConnectionList:
var input meta.ConnectionList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.FirmwareHistoryList:
var input meta.FirmwareHistoryList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.SessionList:
var input meta.SessionList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
case *meta.StreamList:
var input meta.StreamList
if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
return nil, err
}
for _, i := range input {
if err := txn.Insert(list.table, i); err != nil {
return nil, err
}
}
default:
return nil, fmt.Errorf("invalid type found for: %s", list.path)
}
}
txn.Commit()
return &MetaDB{MemDB: db}, nil
}
func TestConsistency(t *testing.T) {
files := map[string]struct {
f string
l meta.List
}{
"connections": {f: "../install/connections.csv", l: &meta.ConnectionList{}},
"cameras": {f: "../install/cameras.csv", l: &meta.InstalledCameraList{}},
"dataloggers": {f: "../install/dataloggers.csv", l: &meta.DeployedDataloggerList{}},
"metsensors": {f: "../install/metsensors.csv", l: &meta.InstalledMetSensorList{}},
"radomes": {f: "../install/radomes.csv", l: &meta.InstalledRadomeList{}},
"receivers": {f: "../install/receivers.csv", l: &meta.DeployedReceiverList{}},
"recorders": {f: "../install/recorders.csv", l: &meta.InstalledRecorderList{}},
"sensors": {f: "../install/sensors.csv", l: &meta.InstalledSensorList{}},
"firmware": {f: "../install/firmware.csv", l: &meta.FirmwareHistoryList{}},
"streams": {f: "../install/streams.csv", l: &meta.StreamList{}},
"networks": {f: "../network/networks.csv", l: &meta.NetworkList{}},
"stations": {f: "../network/stations.csv", l: &meta.StationList{}},
"sites": {f: "../network/sites.csv", l: &meta.SiteList{}},
"marks": {f: "../network/marks.csv", l: &meta.MarkList{}},
"mounts": {f: "../network/mounts.csv", l: &meta.MountList{}},
"gauges": {f: "../network/gauges.csv", l: &meta.GaugeList{}},
"constituents": {f: "../network/constituents.csv", l: &meta.ConstituentList{}},
}
for k, v := range files {
if err := meta.LoadList(v.f, v.l); err != nil {
t.Fatal(err)
}
sort.Sort(v.l)
raw, err := ioutil.ReadFile(v.f)
if err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
if err := csv.NewWriter(&buf).WriteAll(meta.EncodeList(v.l)); err != nil {
t.Fatal(err)
}
if string(raw) != buf.String() {
t.Error(k + ": **** csv file mismatch **** : " + v.f)
file, err := ioutil.TempFile(os.TempDir(), "tst")
if err != nil {
t.Fatal(err)
}
defer os.Remove(file.Name())
file.Write(buf.Bytes())
cmd := exec.Command("diff", "-c", v.f, file.Name())
stdout, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
err = cmd.Start()
if err != nil {
t.Fatal(err)
}
defer cmd.Wait()
diff, err := ioutil.ReadAll(stdout)
if err != nil {
t.Fatal(err)
}
t.Error(string(diff))
}
}
}
func NewMeta(network, install string) (*Meta, error) {
networkMap := make(map[string]meta.Network)
var n meta.NetworkList
if err := meta.LoadList(filepath.Join(network, "networks.csv"), &n); err != nil {
return nil, err
}
for _, v := range n {
networkMap[v.Code] = v
}
stationMap := make(map[string]meta.Station)
var stations meta.StationList
if err := meta.LoadList(filepath.Join(network, "stations.csv"), &stations); err != nil {
return nil, err
}
for _, v := range stations {
stationMap[v.Code] = v
}
connectionMap := make(map[string]meta.ConnectionList)
var connections meta.ConnectionList
if err := meta.LoadList(filepath.Join(install, "connections.csv"), &connections); err != nil {
return nil, err
}
for _, c := range connections {
if _, ok := connectionMap[c.Station]; ok {
connectionMap[c.Station] = append(connectionMap[c.Station], c)
} else {
connectionMap[c.Station] = meta.ConnectionList{c}
}
}
var recorders meta.InstalledRecorderList
if err := meta.LoadList(filepath.Join(install, "recorders.csv"), &recorders); err != nil {
return nil, err
}
for _, r := range recorders {
c := meta.Connection{
Station: r.Station,
Location: r.Location,
Span: meta.Span{
Start: r.Start,
End: r.End,
},
Place: r.Station,
Role: r.Location,
}
if _, ok := connectionMap[c.Station]; ok {
connectionMap[c.Station] = append(connectionMap[c.Station], c)
} else {
connectionMap[c.Station] = meta.ConnectionList{c}
}
}
siteMap := make(map[string]map[string]meta.Site)
var locations meta.SiteList
if err := meta.LoadList(filepath.Join(network, "sites.csv"), &locations); err != nil {
return nil, err
}
for _, l := range locations {
if _, ok := siteMap[l.Station]; !ok {
siteMap[l.Station] = make(map[string]meta.Site)
}
siteMap[l.Station][l.Location] = l
}
streamMap := make(map[string]map[string]meta.StreamList)
var streams meta.StreamList
if err := meta.LoadList(filepath.Join(install, "streams.csv"), &streams); err != nil {
return nil, err
}
for _, s := range streams {
if _, ok := streamMap[s.Station]; !ok {
streamMap[s.Station] = make(map[string]meta.StreamList)
}
if _, ok := streamMap[s.Station][s.Location]; ok {
streamMap[s.Station][s.Location] = append(streamMap[s.Station][s.Location], s)
} else {
streamMap[s.Station][s.Location] = meta.StreamList{s}
}
}
sensorInstalls := make(map[string]meta.InstalledSensorList)
// build sensor installation details
var sensors meta.InstalledSensorList
if err := meta.LoadList(filepath.Join(install, "sensors.csv"), &sensors); err != nil {
return nil, err
}
for _, s := range sensors {
if _, ok := sensorInstalls[s.Station]; ok {
sensorInstalls[s.Station] = append(sensorInstalls[s.Station], s)
} else {
sensorInstalls[s.Station] = meta.InstalledSensorList{s}
}
}
for _, r := range recorders {
if _, ok := sensorInstalls[r.Station]; ok {
sensorInstalls[r.Station] = append(sensorInstalls[r.Station], r.InstalledSensor)
} else {
sensorInstalls[r.Station] = meta.InstalledSensorList{r.InstalledSensor}
}
}
for i, _ := range sensorInstalls {
sort.Sort(sensorInstalls[i])
}
dataloggerDeploys := make(map[string]meta.DeployedDataloggerList)
// where the dataloggers were deployed
var loggers meta.DeployedDataloggerList
if err := meta.LoadList(filepath.Join(install, "dataloggers.csv"), &loggers); err != nil {
return nil, err
}
for _, d := range loggers {
if _, ok := dataloggerDeploys[d.Place]; ok {
dataloggerDeploys[d.Place] = append(dataloggerDeploys[d.Place], d)
} else {
dataloggerDeploys[d.Place] = meta.DeployedDataloggerList{d}
}
}
for _, r := range recorders {
d := meta.DeployedDatalogger{
Install: meta.Install{
Equipment: meta.Equipment{
Make: r.Make,
Model: r.DataloggerModel,
Serial: r.Serial,
},
Span: meta.Span{
Start: r.Start,
End: r.End,
},
},
Place: r.Station,
Role: r.Location,
}
if _, ok := dataloggerDeploys[d.Place]; ok {
dataloggerDeploys[d.Place] = append(dataloggerDeploys[d.Place], d)
} else {
dataloggerDeploys[d.Place] = meta.DeployedDataloggerList{d}
}
}
// sort each datalogger deployment
for i, _ := range dataloggerDeploys {
sort.Sort(dataloggerDeploys[i])
}
return &Meta{
Networks: networkMap,
Stations: stationMap,
Connections: connectionMap,
Sites: siteMap,
Streams: streamMap,
Installs: sensorInstalls,
Deploys: dataloggerDeploys,
}, nil
}
func TestCameras(t *testing.T) {
var cameras meta.InstalledCameraList
t.Log("Load deployed cameras file")
{
if err := meta.LoadList("../install/cameras.csv", &cameras); err != nil {
t.Fatal(err)
}
}
t.Log("Check for cameras installation equipment overlaps")
{
installs := make(map[string]meta.InstalledCameraList)
for _, s := range cameras {
_, ok := installs[s.Model]
if ok {
installs[s.Model] = append(installs[s.Model], s)
} else {
installs[s.Model] = meta.InstalledCameraList{s}
}
}
var keys []string
for k, _ := range installs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := installs[k]
for i, n := 0, len(v); i < n; i++ {
for j := i + 1; j < n; j++ {
switch {
case v[i].Serial != v[j].Serial:
case v[i].End.Before(v[j].Start):
case v[i].Start.After(v[j].End):
case v[i].End.Equal(v[j].Start):
case v[i].Start.Equal(v[j].End):
default:
t.Errorf("cameras %s at %-5s has mount %s overlap between %s and %s",
v[i].Model, v[i].Serial, v[i].Mount, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
}
}
}
}
}
t.Log("Check for missing camera mounts")
{
var mounts meta.MountList
if err := meta.LoadList("../network/mounts.csv", &mounts); err != nil {
t.Fatal(err)
}
keys := make(map[string]interface{})
for _, m := range mounts {
keys[m.Code] = true
}
for _, c := range cameras {
if _, ok := keys[c.Mount]; ok {
continue
}
t.Errorf("unable to find camera mount %-5s", c.Mount)
}
}
var assets meta.AssetList
t.Log("Load camera assets file")
{
if err := meta.LoadList("../assets/cameras.csv", &assets); err != nil {
t.Fatal(err)
}
}
t.Log("Check for camera assets")
{
for _, r := range cameras {
var found bool
for _, a := range assets {
if a.Model != r.Model {
continue
}
if a.Serial != r.Serial {
continue
}
found = true
}
if !found {
t.Errorf("unable to find camera asset: %s [%s]", r.Model, r.Serial)
}
}
}
}
func TestConnections(t *testing.T) {
var connections meta.ConnectionList
t.Log("Load connections file")
if err := meta.LoadList("../install/connections.csv", &connections); err != nil {
t.Fatal(err)
}
for i := 0; i < len(connections); i++ {
for j := i + 1; j < len(connections); j++ {
if connections[i].Station != connections[j].Station {
continue
}
if connections[i].Location != connections[j].Location {
continue
}
if connections[i].Start.After(connections[j].End) {
continue
}
if connections[i].End.Before(connections[j].Start) {
continue
}
t.Errorf("connection overlap: " + strings.Join([]string{
connections[i].Station,
connections[i].Location,
connections[i].Start.String(),
connections[i].End.String(),
}, " "))
}
}
stas := make(map[string]meta.Station)
{
var list meta.StationList
t.Log("Load stations file")
if err := meta.LoadList("../network/stations.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
stas[s.Code] = s
}
}
sites := make(map[string]map[string]meta.Site)
{
var list meta.SiteList
t.Log("Load sites file")
if err := meta.LoadList("../network/sites.csv", &list); err != nil {
t.Fatal(err)
}
for _, s := range list {
if _, ok := sites[s.Station]; !ok {
sites[s.Station] = make(map[string]meta.Site)
}
sites[s.Station][s.Location] = s
}
}
for _, c := range connections {
if _, ok := stas[c.Station]; !ok {
t.Log("unknown connection station: " + c.Station)
} else if s, ok := sites[c.Station]; !ok {
t.Log("unknown connection station: " + c.Station)
} else if _, ok := s[c.Location]; !ok {
t.Log("unknown connection station/location: " + c.Station + "/" + c.Location)
}
if c.Start.After(c.End) {
t.Log("connection span mismatch: " + strings.Join([]string{
c.Station,
c.Location,
c.Start.String(),
"after",
c.End.String(),
}, " "))
}
}
places := make(map[string]string)
{
var list meta.DeployedDataloggerList
t.Log("Load installed dataloggers file")
if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil {
t.Fatal(err)
}
for _, d := range list {
if d.Role != "" {
places[d.Place+"/"+d.Role] = d.Place
} else {
places[d.Place] = d.Place
}
}
}
for _, c := range connections {
if c.Role != "" {
if _, ok := places[c.Place+"/"+c.Role]; !ok {
t.Log("warning: unknown datalogger place/role: " + c.Place + "/" + c.Role)
}
} else {
if _, ok := places[c.Place]; !ok {
t.Log("warning: unknown datalogger place: " + c.Place)
}
}
}
/*
Station Code,Location Code,Datalogger Place,Datalogger Role,Pre Amp,Gain,Start Date,End Date
A11,10,Alfredton,Short Period,false,0,1993-02-01T01:23:00Z,1993-02-21T08:48:00Z
ABAZ,10,Whangaparaoa Navy Base,,false,0,2008-10-13T04:00:00Z,9999-01-01T00:00:00Z
AC1A,10,Wards Pass #1,,false,0,2001-09-12T03:00:01Z,2002-01-22T22:00:00Z
AC2A,10,Wards Pass #2,,false,0,2001-09-13T02:00:01Z,2002-01-22T22:00:00Z
AC3A,10,Acheron #3,,false,0,2001-07-31T23:00:00Z,2002-01-22T23:00:00Z
AC4A,10,Acheron #4,Short Period,false,0,2001-08-01T01:00:00Z,2002-01-22T23:00:00Z
AC5A,10,Wards Pass #5,,false,0,2001-08-01T04:00:00Z,2001-11-27T23:00:00Z
AGA,10,Angora Road,Short Period,false,0,1990-02-20T22:33:00Z,1992-04-07T17:01:00Z
AHAA,10,Ahaura,Short Period,false,0,1995-11-19T17:31:00Z,1995-12-12T01:28:00Z
*/
/*
*/
}
