Beispiel #1
0
func TestConstituents(t *testing.T) {

	gauges := make(map[string]meta.Gauge)
	{
		t.Log("Load installed gauges file")
		var list meta.GaugeList
		if err := meta.LoadList("../network/gauges.csv", &list); err != nil {
			t.Fatal(err)
		}

		for _, g := range list {
			gauges[g.Code] = g
		}
	}

	var constituents meta.ConstituentList
	if err := meta.LoadList("../network/constituents.csv", &constituents); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < len(constituents); i++ {
		for j := i + 1; j < len(constituents); j++ {
			if constituents[i].Gauge == constituents[j].Gauge && constituents[i].Number == constituents[j].Number {
				t.Error("contituent duplication: " + constituents[i].Gauge + "/" + strconv.Itoa(constituents[i].Number))
			}
		}
	}

	for _, c := range constituents {
		if _, ok := gauges[c.Gauge]; !ok {
			t.Error("unknown gauge: " + c.Gauge)
		}

	}
}
Beispiel #2
0
func TestMonuments(t *testing.T) {

	var monuments meta.MonumentList
	t.Log("Load network monuments file")
	{
		if err := meta.LoadList("../network/monuments.csv", &monuments); err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < len(monuments); i++ {
		for j := i + 1; j < len(monuments); j++ {
			if monuments[i].Mark == monuments[j].Mark {
				t.Errorf("monument duplication: " + monuments[i].Mark)
			}
		}
	}

	for _, m := range monuments {
		if m.GroundRelationship > 0.0 {
			t.Errorf("positive monuments ground relationship: %s [%g]", m.Mark, m.GroundRelationship)
		}

		switch m.Type {
		case "Shallow Rod / Braced Antenna Mount":
		case "Wyatt/Agnew Drilled-Braced":
		case "Pillar":
		case "Steel Mast":
		case "Unknown":
		default:
			t.Errorf("unknown monument type: %s [%s]", m.Mark, m.Type)
		}
	}
}
Beispiel #3
0
func TestGauges(t *testing.T) {

	stas := make(map[string]meta.Station)
	{
		var list meta.StationList
		t.Log("Load stations file")
		if err := meta.LoadList("../network/stations.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			stas[s.Code] = s
		}
	}

	var gauges meta.GaugeList
	t.Log("Load installed gauges file")
	{
		if err := meta.LoadList("../network/gauges.csv", &gauges); err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < len(gauges); i++ {
		for j := i + 1; j < len(gauges); j++ {
			if gauges[i].Code == gauges[j].Code {
				t.Errorf("gauge code duplication: " + gauges[i].Code)
			}
			if gauges[i].Number == gauges[j].Number {
				t.Errorf("gauge number duplication: " + gauges[i].Code)
			}
		}
	}

	for _, g := range gauges {
		if _, ok := stas[g.Code]; !ok {
			t.Error("unknown gauge station: " + g.Code)
		}

	}

}
Beispiel #4
0
func TestSites(t *testing.T) {

	var sites meta.SiteList
	t.Log("Load installed sites file")
	{
		if err := meta.LoadList("../network/sites.csv", &sites); err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < len(sites); i++ {
		for j := i + 1; j < len(sites); j++ {
			if sites[i].Station == sites[j].Station && sites[i].Location == sites[j].Location {
				t.Errorf("site duplication: " + sites[i].Station + "/" + sites[i].Location)
			}
		}
	}

}
Beispiel #5
0
func TestNetworks(t *testing.T) {

	var networks meta.NetworkList
	t.Log("Load installed sensors file")
	{
		if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < len(networks); i++ {
		for j := i + 1; j < len(networks); j++ {
			if networks[i].Code == networks[j].Code {
				t.Errorf("network duplication: " + networks[i].Code)
			}
		}
	}

}
Beispiel #6
0
func TestStations(t *testing.T) {

	var stations meta.StationList
	t.Log("Load installed sensors file")
	{
		if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < len(stations); i++ {
		for j := i + 1; j < len(stations); j++ {
			if stations[i].Code == stations[j].Code {
				t.Errorf("station duplication: " + stations[i].Code)
			}
		}
	}

}
Beispiel #7
0
func TestMarks(t *testing.T) {

	var marks meta.MarkList
	t.Log("Load network marks file")
	{
		if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < len(marks); i++ {
		for j := i + 1; j < len(marks); j++ {
			if marks[i].Code == marks[j].Code {
				t.Errorf("mark duplication: " + marks[i].Code)
			}
		}
	}

}
Beispiel #8
0
func TestRecorders(t *testing.T) {
	var recorders meta.InstalledRecorderList

	t.Log("Load installed recorders file")
	{
		if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for recorder installation equipment overlaps")
	{
		installs := make(map[string]meta.InstalledRecorderList)
		for _, s := range recorders {
			_, ok := installs[s.Model]
			if ok {
				installs[s.Model] = append(installs[s.Model], s)

			} else {
				installs[s.Model] = meta.InstalledRecorderList{s}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("recorder %s/%s at %-5s has location %-2s overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for missing recorder stations")
	{
		var stations meta.StationList

		if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
			t.Fatal(err)
		}

		keys := make(map[string]interface{})

		for _, s := range stations {
			keys[s.Code] = true
		}

		for _, s := range recorders {
			if _, ok := keys[s.Station]; ok {
				continue
			}
			t.Errorf("unable to find recorder installed station %-5s", s.Station)
		}
	}

	var assets meta.AssetList
	t.Log("Load recorder assets file")
	{
		if err := meta.LoadList("../assets/recorders.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for recorder assets")
	{
		for _, r := range recorders {
			model := r.DataloggerModel
			if r.DataloggerModel != r.Model {
				model = strings.Join([]string{r.DataloggerModel, r.Model}, " ")
			}

			var found bool
			for _, a := range assets {
				if a.Model != model {
					continue
				}
				if a.Serial != r.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find recorders asset: %s [%s]", model, r.Serial)
			}
		}
	}

}
Beispiel #9
0
// TestStationsProto creates a Protobuf file of Stations.
func TestStationsProto(t *testing.T) {
	var networks meta.NetworkList

	if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
		t.Error(err)
	}

	var net = make(map[string]*delta.Network)

	for _, v := range networks {
		n := delta.Network{
			Code:        v.Code,
			External:    v.External,
			Description: v.Description,
			Restricted:  v.Restricted,
		}

		net[v.Code] = &n
	}

	var stations meta.StationList
	if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
		t.Error(err)
	}

	if len(stations) == 0 {
		t.Error("zero length stations list.")
	}

	var s delta.Stations
	s.Stations = make(map[string]*delta.Station)

	for _, v := range stations {
		pt := delta.Point{
			Longitude: v.Longitude,
			Latitude:  v.Latitude,
			Elevation: v.Elevation,
			Datum:     v.Datum,
		}

		sp := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}

		st := delta.Station{
			Code:    v.Code,
			Name:    v.Name,
			Network: net[v.Network],
			Point:   &pt,
			Span:    &sp,
		}

		s.Stations[st.Code] = &st
	}

	b, err := proto.Marshal(&s)
	if err != nil {
		t.Error(err)
	}

	t.Log(s)

	if err := os.MkdirAll(apiDir, 0777); err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(apiDir+"/stations.pb", b, 0644); err != nil {
		t.Error(err)
	}
}
Beispiel #10
0
// TestMarksProto creates protobuf and JSON files of Marks.
// These are pushed to S3 (by Travis) for use in api.geonet.org.nz
// Three files are created:
// marks.pb - fully hydrated protobuf will all GNSS Mark information.
// marks.json - JSON version of marks.pb (for use in browsers).
// marks.geojson - GeoJSON of Mark locations.
func TestMarksProto(t *testing.T) {
	var networks meta.NetworkList

	if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
		t.Error(err)
	}

	var net = make(map[string]*delta.Network)

	for _, v := range networks {
		n := delta.Network{
			Code:        v.Code,
			External:    v.External,
			Description: v.Description,
			Restricted:  v.Restricted,
		}

		net[v.Code] = &n
	}

	var marks meta.MarkList
	if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
		t.Error(err)
	}

	if len(marks) == 0 {
		t.Error("zero length mark list.")
	}

	var m delta.Marks

	m.Marks = make(map[string]*delta.Mark)

	for _, v := range marks {
		pt := delta.Point{
			Longitude: v.Longitude,
			Latitude:  v.Latitude,
			Elevation: v.Elevation,
			Datum:     v.Datum,
		}

		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}

		mk := delta.Mark{
			Code:    v.Code,
			Name:    v.Name,
			Network: net[v.Network],
			Point:   &pt,
			Span:    &s,
		}

		m.Marks[mk.Code] = &mk
	}

	var monuments meta.MonumentList
	if err := meta.LoadList("../network/monuments.csv", &monuments); err != nil {
		t.Error(err)
	}

	for _, v := range monuments {
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}

		mn := delta.Monument{
			DomesNumber:        v.DomesNumber,
			MarkType:           v.MarkType,
			Type:               v.Type,
			GroundRelationship: v.GroundRelationship,
			FoundationType:     v.FoundationType,
			FoundationDepth:    v.FoundationDepth,
			Bedrock:            v.Bedrock,
			Geology:            v.Geology,
			Span:               &s,
		}
		if _, ok := m.Marks[v.Mark]; ok {
			m.Marks[v.Mark].Monument = &mn
		}
	}

	var antennas meta.InstalledAntennaList
	if err := meta.LoadList("../install/antennas.csv", &antennas); err != nil {
		t.Error(err)
	}

	for _, v := range antennas {
		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
		}
		o := delta.Offset{
			Vertical: v.Vertical,
			North:    v.North,
			East:     v.East,
		}
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		a := delta.InstalledAntenna{
			Equipment: &e,
			Offset:    &o,
			Span:      &s,
			Azimuth:   v.Azimuth,
		}

		if _, ok := m.Marks[v.Mark]; ok {
			m.Marks[v.Mark].InstalledAntenna = append(m.Marks[v.Mark].InstalledAntenna, &a)
		}
	}

	var radomes meta.InstalledRadomeList
	if err := meta.LoadList("../install/radomes.csv", &radomes); err != nil {
		t.Error(err)
	}

	for _, v := range radomes {
		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
		}
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		r := delta.InstalledRadome{
			Equipment: &e,
			Span:      &s,
		}
		if _, ok := m.Marks[v.Mark]; ok {
			m.Marks[v.Mark].InstalledRadome = append(m.Marks[v.Mark].InstalledRadome, &r)
		}
	}

	var firmwares meta.FirmwareHistoryList
	if err := meta.LoadList("../install/firmware.csv", &firmwares); err != nil {
		t.Error(err)
	}

	var fw = make(map[delta.Equipment]delta.Receiver)

	for _, v := range firmwares {
		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
		}
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		f := delta.Firmware{
			Version: v.Version,
			Notes:   v.Notes,
			Span:    &s,
		}

		rx := fw[e]
		rx.Equipment = &e
		rx.Firmware = append(rx.Firmware, &f)
		fw[e] = rx
	}

	var receivers meta.DeployedReceiverList
	if err := meta.LoadList("../install/receivers.csv", &receivers); err != nil {
		t.Error(err)
	}

	for _, v := range receivers {
		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
		}
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}

		rx, ok := fw[e]
		if !ok {
			t.Errorf("no firware for %v", e)
		}

		d := delta.DeployedReceiver{
			Receiver: &rx,
			Span:     &s,
		}

		if _, ok := m.Marks[v.Mark]; ok {
			m.Marks[v.Mark].DeployedReceiver = append(m.Marks[v.Mark].DeployedReceiver, &d)
		}
	}

	var sessions meta.SessionList
	if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil {
		t.Error(err)
	}

	for _, v := range sessions {
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		se := delta.Session{
			Operator:        v.Operator,
			Agency:          v.Agency,
			Model:           v.Model,
			SatelliteSystem: v.SatelliteSystem,
			Interval:        v.Interval.Nanoseconds(),
			ElevationMask:   v.ElevationMask,
			HeaderComment:   v.HeaderComment,
			Span:            &s,
		}
		if _, ok := m.Marks[v.Mark]; ok {
			m.Marks[v.Mark].Session = append(m.Marks[v.Mark].Session, &se)
		}
	}

	var metsensors meta.InstalledMetSensorList
	if err := meta.LoadList("../install/metsensors.csv", &metsensors); err != nil {
		t.Error(err)
	}

	for _, v := range metsensors {
		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
		}
		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		p := delta.Point{
			Longitude: v.Longitude,
			Latitude:  v.Latitude,
			Elevation: v.Elevation,
			Datum:     v.Datum,
		}
		ms := delta.InstalledMetSensor{
			Equipment:  &e,
			Span:       &s,
			Point:      &p,
			IMSComment: v.IMSComment,
		}
		if _, ok := m.Marks[v.Mark]; ok {
			m.Marks[v.Mark].InstalledMetSensor = append(m.Marks[v.Mark].InstalledMetSensor, &ms)
		}
	}

	// output files

	if err := os.MkdirAll(apiDir, 0777); err != nil {
		t.Error(err)
	}

	b, err := proto.Marshal(&m)
	if err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(apiDir+"/marks.pb", b, 0644); err != nil {
		t.Error(err)
	}

	b, err = json.Marshal(&m)
	if err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(apiDir+"/marks.json", b, 0644); err != nil {
		t.Error(err)
	}

	// GeoJSON files of site for each sensor type.
	// This is similar to the sensor type output in stations_test.go
	// There is no other sensor type lookup so handle that switching here.
	// There is fractionally more work as this is two different installed types
	// (antenna and metsensor).

	out := map[string]*bytes.Buffer{
		"metsensor":  &bytes.Buffer{},
		"gpsantenna": &bytes.Buffer{},
	}

	for k := range out {
		out[k].WriteString(`{"type": "FeatureCollection","features": [`)
	}

	for _, mark := range m.Marks {
		if mark.GetSpan() == nil || mark.GetPoint() == nil || mark.GetNetwork() == nil {
			continue
		}

		if mark.InstalledAntenna != nil && len(mark.InstalledAntenna) > 0 {
			for _, v := range mark.InstalledAntenna {
				writeAntennaProps(mark, v, out["gpsantenna"])
			}
		}

		if mark.InstalledMetSensor != nil && len(mark.InstalledMetSensor) > 0 {
			for _, v := range mark.InstalledMetSensor {
				writeMetsensorProps(mark, v, out["metsensor"])
			}
		}
	}

	for k := range out {
		out[k].WriteString(`]}`)

		fn := apiDir + "/" + k + ".geojson"

		if err := ioutil.WriteFile(fn, out[k].Bytes(), 0644); err != nil {
			t.Error(err)
		}
	}
}
Beispiel #11
0
func TestSensors(t *testing.T) {
	var installed meta.InstalledSensorList
	t.Log("Load installed sensors file")
	{
		if err := meta.LoadList("../install/sensors.csv", &installed); err != nil {
			t.Fatal(err)
		}
		sort.Sort(installed)
	}

	t.Log("Check for missing sensors")
	{
		var sensors meta.AssetList
		if err := meta.LoadList("../assets/sensors.csv", &sensors); err != nil {
			t.Fatal(err)
		}
		sort.Sort(sensors)
		for _, i := range installed {
			n := sort.Search(len(sensors), func(j int) bool { return !sensors[j].Equipment.Less(i.Equipment) })
			if n < 0 || i.Equipment.Less(sensors[n].Equipment) {
				t.Errorf("unable to find sensor: %s", i.String())
			}
		}
	}

	t.Log("Check for sensor installation overlaps")
	{
		installs := make(map[string]meta.InstalledSensorList)
		for _, s := range installed {
			_, ok := installs[s.Model]
			if ok {
				installs[s.Model] = append(installs[s.Model], s)

			} else {
				installs[s.Model] = meta.InstalledSensorList{s}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("sensor %s/%s at %-5s has location %-2s overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	/*
		t.Log("Check for missing sensor stations")
		{
			var stations meta.StationList

			if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
				t.Fatal(err)
			}

			sort.Sort(stations)

			for _, i := range installed {
				n := sort.Search(len(stations), func(j int) bool { return !(stations[j].Code < i.StationCode) })
				if n < 0 || i.StationCode != stations[n].Code {
					t.Errorf("unable to find station: %s", i.StationCode)
				} else {
					if i.Start.Before(stations[n].Start) {
						t.Errorf("installed sensor before station has been opened: %s: %s (%s %s)", i.String(), i.Start.String(), stations[n].Code, stations[n].Start.String())
					}
					if i.End.After(stations[n].End) {
						t.Errorf("installed sensor after station has been closed: %s: %s (%s %s)", i.String(), i.End.String(), stations[n].Code, stations[n].End.String())
					}
				}
			}
		}

		t.Log("Check for missing sensor sites")
		{
			var sites meta.SiteList

			if err := meta.LoadList("../network/sites.csv", &sites); err != nil {
				t.Fatal(err)
			}

			sort.Sort(sites)

			for _, i := range installed {
				n := sort.Search(len(sites), func(j int) bool {
					if sites[j].StationCode > i.StationCode {
						return true
					}
					if sites[j].StationCode < i.StationCode {
						return false
					}
					return !(sites[j].LocationCode < i.LocationCode)
				})
				if n < 0 || i.StationCode != sites[n].StationCode || i.LocationCode != sites[n].LocationCode {
					t.Errorf("unable to find site: %s/%s", i.StationCode, i.LocationCode)
				} else {
					if i.Start.Before(sites[n].Start) {
						t.Errorf("installed sensor before site has been opened: %s: %s (%s/%s %s)", i.String(), i.Start.String(), sites[n].StationCode, sites[n].LocationCode, sites[n].Start.String())
					}
					if i.End.After(sites[n].End) {
						t.Errorf("installed sensor after site has been closed: %s: %s (%s/%s %s)", i.String(), i.End.String(), sites[n].StationCode, sites[n].LocationCode, sites[n].End.String())
					}
				}
			}
		}
	*/

	for _, i := range installed {
		if i.Orientation.Azimuth < -360.0 || i.Orientation.Azimuth > 360.0 {
			t.Errorf("installed sensor has invalid orientation azimuth: %s [%g]", i.String(), i.Orientation.Azimuth)
		}
		if i.Orientation.Dip < -90.0 || i.Orientation.Dip > 90.0 {
			t.Errorf("installed sensor has invalid orientation dip: %s [%g]", i.String(), i.Orientation.Dip)
		}
	}

	var assets meta.AssetList
	t.Log("Load receiver sensors file")
	{
		if err := meta.LoadList("../assets/sensors.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for sensor assets")
	{
		for _, s := range installed {
			var found bool
			for _, a := range assets {
				if a.Model != s.Model {
					continue
				}
				if a.Serial != s.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find sensor asset: %s [%s]", s.Model, s.Serial)
			}
		}
	}

}
Beispiel #12
0
func TestReceivers(t *testing.T) {

	var receivers meta.DeployedReceiverList
	t.Log("Load deployed receivers file")
	{
		if err := meta.LoadList("../install/receivers.csv", &receivers); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for particular receiver installation overlaps")
	{
		installs := make(map[string]meta.DeployedReceiverList)
		for _, s := range receivers {
			if _, ok := installs[s.Model]; !ok {
				installs[s.Model] = meta.DeployedReceiverList{}
			}
			installs[s.Model] = append(installs[s.Model], s)
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("receiver %s [%s] at %s has overlap with %s between times %s and %s",
							v[i].Model, v[i].Serial, v[i].Mark, v[j].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for receiver sites installation equipment overlaps")
	{
		installs := make(map[string]meta.DeployedReceiverList)
		for _, s := range receivers {
			if _, ok := installs[s.Mark]; !ok {
				installs[s.Mark] = meta.DeployedReceiverList{}
			}
			installs[s.Mark] = append(installs[s.Model], s)
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("receivers %s [%s] / %s [%s] at %s has overlap between %s and %s",
							v[i].Model, v[i].Serial, v[j].Model, v[j].Serial, v[i].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for missing receiver marks")
	{
		var marks meta.MarkList

		if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
			t.Fatal(err)
		}

		keys := make(map[string]interface{})

		for _, m := range marks {
			keys[m.Code] = true
		}

		for _, r := range receivers {
			if _, ok := keys[r.Mark]; ok {
				continue
			}
			t.Errorf("unable to find receiver mark %-5s", r.Mark)
		}
	}

	var assets meta.AssetList
	t.Log("Load receiver assets file")
	{
		if err := meta.LoadList("../assets/receivers.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for receiver assets")
	{
		for _, r := range receivers {
			var found bool
			for _, a := range assets {
				if a.Model != r.Model {
					continue
				}
				if a.Serial != r.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find receiver asset: %s [%s]", r.Model, r.Serial)
			}
		}
	}

}
Beispiel #13
0
func TestConnections(t *testing.T) {

	var connections meta.ConnectionList

	t.Log("Load connections file")
	if err := meta.LoadList("../install/connections.csv", &connections); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < len(connections); i++ {
		for j := i + 1; j < len(connections); j++ {
			if connections[i].Station != connections[j].Station {
				continue
			}
			if connections[i].Location != connections[j].Location {
				continue
			}
			if connections[i].Start.After(connections[j].End) {
				continue
			}
			if connections[i].End.Before(connections[j].Start) {
				continue
			}
			t.Errorf("connection overlap: " + strings.Join([]string{
				connections[i].Station,
				connections[i].Location,
				connections[i].Start.String(),
				connections[i].End.String(),
			}, " "))
		}
	}

	stas := make(map[string]meta.Station)
	{
		var list meta.StationList
		t.Log("Load stations file")
		if err := meta.LoadList("../network/stations.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			stas[s.Code] = s
		}
	}

	sites := make(map[string]map[string]meta.Site)
	{
		var list meta.SiteList
		t.Log("Load sites file")
		if err := meta.LoadList("../network/sites.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			if _, ok := sites[s.Station]; !ok {
				sites[s.Station] = make(map[string]meta.Site)
			}
			sites[s.Station][s.Location] = s
		}
	}

	for _, c := range connections {
		if _, ok := stas[c.Station]; !ok {
			t.Log("unknown connection station: " + c.Station)
		} else if s, ok := sites[c.Station]; !ok {
			t.Log("unknown connection station: " + c.Station)
		} else if _, ok := s[c.Location]; !ok {
			t.Log("unknown connection station/location: " + c.Station + "/" + c.Location)
		}
		if c.Start.After(c.End) {
			t.Log("connection span mismatch: " + strings.Join([]string{
				c.Station,
				c.Location,
				c.Start.String(),
				"after",
				c.End.String(),
			}, " "))
		}
	}

	places := make(map[string]string)
	{
		var list meta.DeployedDataloggerList
		t.Log("Load installed dataloggers file")
		if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, d := range list {
			if d.Role != "" {
				places[d.Place+"/"+d.Role] = d.Place
			} else {
				places[d.Place] = d.Place
			}
		}
	}

	for _, c := range connections {
		if c.Role != "" {
			if _, ok := places[c.Place+"/"+c.Role]; !ok {
				t.Log("warning: unknown datalogger place/role: " + c.Place + "/" + c.Role)
			}
		} else {
			if _, ok := places[c.Place]; !ok {
				t.Log("warning: unknown datalogger place: " + c.Place)
			}
		}
	}

	var missing []meta.Connection

	var assets = make(map[string]meta.Asset)
	{
		var list meta.AssetList
		t.Log("Load datalogger assets file")
		if err := meta.LoadList("../assets/dataloggers.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, d := range list {
			assets[d.Model+":::"+d.Serial] = d
		}
		t.Log("Load sensor assets file")
		if err := meta.LoadList("../assets/sensors.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			assets[s.Model+":::"+s.Serial] = s
		}
	}

	var sensors []meta.InstalledSensor
	{
		var list meta.InstalledSensorList
		t.Log("Load stations file")
		if err := meta.LoadList("../install/sensors.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			sensors = append(sensors, s)
		}
	}

	for _, s := range sensors {
		if a, ok := assets[s.Model+":::"+s.Serial]; !ok || a.Number == "" {
			continue
		}
		if s.End.Before(time.Now()) {
			continue
		}
		var handled bool
		for _, c := range connections {
			if c.Station != s.Station || c.Location != s.Location {
				continue
			}
			if c.Start.After(s.End) || c.End.Before(s.Start) {
				continue
			}
			handled = true
		}
		if !handled {
			var place string
			if p, ok := stas[s.Station]; ok {
				place = p.Name
			}
			missing = append(missing, meta.Connection{
				Station:  s.Station,
				Location: s.Location,
				Place:    place,
				Span: meta.Span{
					Start: s.Start,
					End:   s.End,
				},
			})
			t.Errorf("no current connection defined for sensor: %s [%s/%s] %s %s", s.String(), s.Station, s.Location, s.Start, s.End)
		}
	}

	var dataloggers []meta.DeployedDatalogger
	{
		var list meta.DeployedDataloggerList
		t.Log("Load stations file")
		if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, d := range list {
			dataloggers = append(dataloggers, d)
		}
	}

	for _, d := range dataloggers {
		if a, ok := assets[d.Model+":::"+d.Serial]; !ok || a.Number == "" {
			continue
		}
		if d.End.Before(time.Now()) {
			continue
		}
		var handled bool
		for _, c := range connections {
			if c.Place != d.Place || c.Role != d.Role {
				continue
			}
			if c.Start.After(d.End) || c.End.Before(d.Start) {
				continue
			}
			handled = true
		}
		if !handled {
			s, l := "XXXX", "LL"
			for k, v := range stas {
				if v.Name == d.Place {
					s = k
				}
			}

			missing = append(missing, meta.Connection{
				Station:  s,
				Location: l,
				Place:    d.Place,
				Role:     d.Role,
				Span: meta.Span{
					Start: d.Start,
					End:   d.End,
				},
			})

			t.Errorf("no current connection defined for datalogger: %s [%s/%s] %s %s", d.String(), d.Place, d.Role, d.Start, d.End)
		}
	}

	if len(missing) > 0 {
		sort.Sort(meta.ConnectionList(missing))
		t.Log("\n" + string(meta.MarshalList(meta.ConnectionList(missing))))
	}

}
Beispiel #14
0
func TestStreams(t *testing.T) {

	var streams meta.StreamList

	t.Log("Load streams file")
	if err := meta.LoadList("../install/streams.csv", &streams); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < len(streams); i++ {
		for j := i + 1; j < len(streams); j++ {
			if streams[i].Station != streams[j].Station {
				continue
			}
			if streams[i].Location != streams[j].Location {
				continue
			}
			if streams[i].Start.After(streams[j].End) {
				continue
			}
			if streams[i].End.Before(streams[j].Start) {
				continue
			}
			if streams[i].SamplingRate != streams[j].SamplingRate {
				continue
			}
			t.Errorf("stream overlap: " + strings.Join([]string{
				streams[i].Station,
				streams[i].Location,
				streams[i].Start.String(),
				streams[i].End.String(),
			}, " "))
		}
	}

	stas := make(map[string]meta.Station)
	{
		var list meta.StationList
		t.Log("Load stations file")
		if err := meta.LoadList("../network/stations.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			stas[s.Code] = s
		}
	}

	sites := make(map[string]map[string]meta.Site)
	{
		var list meta.SiteList
		t.Log("Load sites file")
		if err := meta.LoadList("../network/sites.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			if _, ok := sites[s.Station]; !ok {
				sites[s.Station] = make(map[string]meta.Site)
			}
			sites[s.Station][s.Location] = s
		}
	}

	for _, c := range streams {
		if _, ok := stas[c.Station]; !ok {
			t.Log("unknown stream station: " + c.Station)
		} else if s, ok := sites[c.Station]; !ok {
			t.Log("unknown stream station: " + c.Station)
		} else if _, ok := s[c.Location]; !ok {
			t.Log("unknown stream station/location: " + c.Station + "/" + c.Location)
		}
		if c.Start.After(c.End) {
			t.Log("stream span mismatch: " + strings.Join([]string{
				c.Station,
				c.Location,
				c.Start.String(),
				"after",
				c.End.String(),
			}, " "))
		}
	}
}
Beispiel #15
0
func TestAntennas(t *testing.T) {

	var antennas meta.InstalledAntennaList
	t.Log("Load deployed antennas file")
	{
		if err := meta.LoadList("../install/antennas.csv", &antennas); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for antenna installation equipment overlaps")
	{
		installs := make(map[string]meta.InstalledAntennaList)
		for _, s := range antennas {
			if _, ok := installs[s.Model]; !ok {
				installs[s.Model] = meta.InstalledAntennaList{}
			}
			installs[s.Model] = append(installs[s.Model], s)
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("antennas %s [%s] at %s has overlap at %s between %s and %s",
							v[i].Model, v[i].Serial, v[i].Mark, v[j].Mark, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for antenna installation mark overlaps")
	{
		installs := make(map[string]meta.InstalledAntennaList)
		for _, s := range antennas {
			if _, ok := installs[s.Mark]; !ok {
				installs[s.Mark] = meta.InstalledAntennaList{}
			}
			installs[s.Mark] = append(installs[s.Mark], s)
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("antennas %s [%s] and %s [%s] at %s has overlap between %s and %s",
							v[i].Model, v[i].Serial, v[j].Model, v[j].Serial, v[i].Mark,
							v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for missing antenna marks")
	{
		var marks meta.MarkList

		if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
			t.Fatal(err)
		}

		keys := make(map[string]interface{})

		for _, m := range marks {
			keys[m.Code] = true
		}

		for _, c := range antennas {
			if _, ok := keys[c.Mark]; ok {
				continue
			}
			t.Errorf("unable to find antenna mark %-5s", c.Mark)
		}
	}

	var assets meta.AssetList
	t.Log("Load antenna assets file")
	{
		if err := meta.LoadList("../assets/antennas.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for antenna assets")
	{
		for _, r := range antennas {
			var found bool
			for _, a := range assets {
				if a.Model != r.Model {
					continue
				}
				if a.Serial != r.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find antenna asset: %s [%s]", r.Model, r.Serial)
			}
		}
	}

	var sessions meta.SessionList
	t.Log("Load session list")
	{
		if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check sessions exist ...")
	{
		for _, r := range antennas {
			var found bool
			for _, s := range sessions {
				if s.End.Before(r.Start) {
					continue
				}
				if s.Start.After(r.End) {
					continue
				}
				found = true
			}
			if !found {
				t.Log(r)
			}
		}
	}
}
Beispiel #16
0
func TestFirmware(t *testing.T) {

	var firmwares meta.FirmwareHistoryList
	t.Log("Load firmware history file")
	{
		if err := meta.LoadList("../install/firmware.csv", &firmwares); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for firmware history overlaps")
	{
		installs := make(map[string]meta.FirmwareHistoryList)
		for _, s := range firmwares {
			_, ok := installs[s.Model]
			if ok {
				installs[s.Model] = append(installs[s.Model], s)

			} else {
				installs[s.Model] = meta.FirmwareHistoryList{s}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("firmware %s / %s has overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	var assets meta.AssetList
	t.Log("Load firmware assets file")
	{
		if err := meta.LoadList("../assets/receivers.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for firmware receiver assets")
	{
		for _, r := range firmwares {
			var found bool
			for _, a := range assets {
				if a.Model != r.Model {
					continue
				}
				if a.Serial != r.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find firmware receiver asset: %s [%s]", r.Model, r.Serial)
			}
		}
	}

}
Beispiel #17
0
func main() {

	var verbose bool
	flag.BoolVar(&verbose, "verbose", false, "make noise")

	var output string
	flag.StringVar(&output, "output", "output", "output directory")

	var logs string
	flag.StringVar(&logs, "logs", "logs", "logs output directory")

	var network string
	flag.StringVar(&network, "network", "../../network", "base network directory")

	var install string
	flag.StringVar(&install, "install", "../../install", "base install directory")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "Build GNSS SiteLog XML files from delta meta information\n")
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "Usage:\n")
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "  %s [options]\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "Options:\n")
		fmt.Fprintf(os.Stderr, "\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
	}

	flag.Parse()

	var tplFuncMap template.FuncMap = template.FuncMap{
		"empty": func(d, s string) string {
			if s != "" {
				return s
			}
			return d
		},
		"tolower": func(s string) string {
			switch t := strings.ToLower(s); t {
			case "wyatt/agnew drilled-braced":
				return "Deep Wyatt/Agnew drilled-braced"
			default:
				return t
			}
		},
		"lines": func(p, s string) string {
			switch s {
			case "":
				return s
			default:
				return strings.Join(strings.Split(s, "\n"), "\n"+p)
			}
		},
		"plus": func(n int) string {
			return fmt.Sprintf("%-2s", strconv.Itoa(n+1))
		},
		"lat": func(s string) string {
			if f, err := strconv.ParseFloat(s, 64); err == nil {
				m := math.Abs(f-float64(int(f))) * 60.0
				return fmt.Sprintf("%+3d%02d%05.2f", int(f), int(m), (m-float64(int(m)))*60.0)
			}
			return ""
		},
		"lon": func(s string) string {
			if f, err := strconv.ParseFloat(s, 64); err == nil {
				m := math.Abs(f-float64(int(f))) * 60.0
				return fmt.Sprintf("%+3d%02d%05.2f", int(f), int(m), (m-float64(int(m)))*60.0)
			}
			return ""
		},
	}

	tmpl, err := template.New("").Funcs(tplFuncMap).Parse(sitelogTemplate)
	if err != nil {
		log.Fatalf("error: unable to compile template: %v", err)
	}

	var firmwareHistoryList meta.FirmwareHistoryList
	if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil {
		log.Fatalf("error: unable to load firmware history: %v", err)
	}

	firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory)
	for _, i := range firmwareHistoryList {
		if _, ok := firmwareHistory[i.Model]; !ok {
			firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory)
		}
		firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i)
	}

	for j, _ := range firmwareHistory {
		for k, _ := range firmwareHistory[j] {
			sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k]))
		}
	}

	var installedAntennaList meta.InstalledAntennaList
	if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil {
		log.Fatalf("error: unable to load antenna installs: %v", err)
	}

	installedAntenna := make(map[string][]meta.InstalledAntenna)
	for _, i := range installedAntennaList {
		installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i)
	}
	for i, _ := range installedAntenna {
		sort.Sort(meta.InstalledAntennaList(installedAntenna[i]))
	}

	var deployedReceiverList meta.DeployedReceiverList
	if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil {
		log.Fatalf("error: unable to load receiver installs: %v", err)
	}

	deployedReceivers := make(map[string][]meta.DeployedReceiver)
	for _, i := range deployedReceiverList {
		deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i)
	}
	for i, _ := range deployedReceivers {
		sort.Sort(meta.DeployedReceiverList(deployedReceivers[i]))
	}

	var installedRadomeList meta.InstalledRadomeList
	if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil {
		log.Fatalf("error: unable to load radome installs: %v", err)
	}

	installedRadomes := make(map[string][]meta.InstalledRadome)
	for _, i := range installedRadomeList {
		installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i)
	}
	for i, _ := range installedRadomes {
		sort.Sort(meta.InstalledRadomeList(installedRadomes[i]))
	}

	var installedMetSensorList meta.InstalledMetSensorList
	if err := meta.LoadList(filepath.Join(install, "metsensors.csv"), &installedMetSensorList); err != nil {
		log.Fatalf("error: unable to load metsensors list: %v", err)
	}

	installedMetSensors := make(map[string][]meta.InstalledMetSensor)
	for _, i := range installedMetSensorList {
		installedMetSensors[i.Mark] = append(installedMetSensors[i.Mark], i)
	}
	for i, _ := range installedMetSensors {
		sort.Sort(meta.InstalledMetSensorList(installedMetSensors[i]))
	}

	var markList meta.MarkList
	if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil {
		log.Fatalf("error: unable to load mark list: %v", err)
	}

	var sessionList meta.SessionList
	if err := meta.LoadList(filepath.Join(install, "sessions.csv"), &sessionList); err != nil {
		log.Fatalf("error: unable to load session list: %v", err)
	}

	sessions := make(map[string][]meta.Session)
	for _, s := range sessionList {
		sessions[s.Mark] = append(sessions[s.Mark], s)
	}

	var monumentList meta.MonumentList
	if err := meta.LoadList(filepath.Join(network, "monuments.csv"), &monumentList); err != nil {
		log.Fatalf("error: unable to load monument list: %v", err)
	}
	monuments := make(map[string]meta.Monument)
	for _, m := range monumentList {
		monuments[m.Mark] = m
	}

	for _, m := range markList {

		if _, ok := monuments[m.Code]; !ok {
			continue
		}

		if _, ok := sessions[m.Code]; !ok {
			continue
		}
		if _, ok := installedAntenna[m.Code]; !ok {
			continue
		}
		if _, ok := deployedReceivers[m.Code]; !ok {
			continue
		}

		var receivers []GnssReceiver
		var antennas []GnssAntenna
		var metsensors []GnssMetSensor

		for _, m := range installedMetSensors[m.Reference.Code] {
			var session *meta.Session
			for i, s := range sessions[m.Mark] {
				if m.Start.After(s.End) || m.End.Before(s.Start) {
					continue
				}
				session = &sessions[m.Mark][i]
				break
			}
			if session == nil {
				continue
			}
			metsensors = append(metsensors, GnssMetSensor{
				Manufacturer:         m.Make,
				MetSensorModel:       m.Model,
				SerialNumber:         m.Serial,
				DataSamplingInterval: "360 sec",
				EffectiveDates:       "2000-02-05/CCYY-MM-DD",
				Notes:                "",
			})
		}

		for _, a := range installedAntenna[m.Code] {
			var session *meta.Session
			for i, s := range sessions[m.Code] {
				if a.Start.After(s.End) || a.End.Before(s.Start) {
					continue
				}
				session = &sessions[m.Code][i]
				break
			}
			if session == nil {
				continue
			}

			radome := "NONE"
			serial := ""
			if _, ok := installedRadomes[m.Code]; ok {
				for _, v := range installedRadomes[m.Code] {
					if v.Start.After(a.End) || v.End.Before(a.Start) {
						continue
					}
					radome = v.Model
					serial = v.Serial
				}
			}

			antennas = append(antennas, GnssAntenna{
				AntennaType:            a.Model,
				SerialNumber:           a.Serial,
				AntennaReferencePoint:  "BAM",
				MarkerArpUpEcc:         strconv.FormatFloat(a.Vertical, 'f', 4, 64),
				MarkerArpNorthEcc:      strconv.FormatFloat(a.North, 'f', 4, 64),
				MarkerArpEastEcc:       strconv.FormatFloat(a.East, 'f', 4, 64),
				AlignmentFromTrueNorth: "0",
				AntennaRadomeType:      radome,
				RadomeSerialNumber:     serial,
				AntennaCableType:       "",
				AntennaCableLength:     "",
				DateInstalled:          a.Start.Format(DateTimeFormat),
				DateRemoved: func() string {
					if time.Now().After(a.End) {
						return a.End.Format(DateTimeFormat)
					} else {
						return ""
					}
				}(),
				Notes: "",
			})
		}

		for _, r := range deployedReceivers[m.Code] {
			if _, ok := firmwareHistory[r.Model]; ok {
				if _, ok := firmwareHistory[r.Model][r.Serial]; ok {
					for i, _ := range firmwareHistory[r.Model][r.Serial] {

						v := firmwareHistory[r.Model][r.Serial][len(firmwareHistory[r.Model][r.Serial])-i-1]
						if v.End.Before(r.Start) || v.Start.After(r.End) {
							continue
						}

						var session *meta.Session
						for i, s := range sessions[m.Code] {
							if r.Start.After(s.End) || r.End.Before(s.Start) {
								continue
							}
							if v.Start.After(s.End) || v.End.Before(s.Start) {
								continue
							}
							session = &sessions[m.Code][i]
							break
						}
						if session == nil {
							continue
						}

						start := r.Start
						/*
							if start.Before(s.Start) {
								start = s.Start
							}
						*/
						if start.Before(v.Start) {
							start = v.Start
						}

						end := r.End
						/*
							if end.After(s.End) {
								end = s.End
							}
						*/
						if end.After(v.End) {
							end = v.End
						}

						receivers = append(receivers, GnssReceiver{
							ReceiverType:           r.Model,
							SatelliteSystem:        session.SatelliteSystem,
							SerialNumber:           r.Serial,
							FirmwareVersion:        v.Version,
							ElevationCutoffSetting: strconv.FormatFloat(session.ElevationMask, 'g', -1, 64),
							DateInstalled:          start.Format(DateTimeFormat),
							/*
								DateInstalled: func() string {
									if v.Start.Before(r.Start) {
										return r.Start.Format(DateTimeFormat)
									} else {
										return v.Start.Format(DateTimeFormat)
									}
								}(),
							*/
							DateRemoved: func() string {
								/*
									if v.End.After(r.End) {
										if time.Now().After(r.End) {
											return r.End.Format(DateTimeFormat)
										} else {
											return ""
										}
									} else {
										if time.Now().After(v.End) {
											return v.End.Format(DateTimeFormat)
										} else {
											return ""
										}
									}
								*/
								if time.Now().After(end) {
									return end.Format(DateTimeFormat)
								} else {
									return ""
								}
							}(),
							TemperatureStabilization: "",
							Notes: "",
						})
					}
				}
			}
		}

		sort.Sort(GnssReceivers(receivers))
		sort.Sort(GnssAntennas(antennas))

		monument := monuments[m.Code]

		X, Y, Z := WGS842ITRF(m.Latitude, m.Longitude, m.Elevation)

		x := SiteLog{

			EquipNameSpace:   equipNameSpace,
			ContactNameSpace: contactNameSpace,
			MiNameSpace:      miNameSpace,
			LiNameSpace:      liNameSpace,
			XmlNameSpace:     xmlNameSpace,
			XsiNameSpace:     xsiNameSpace,
			SchemaLocation:   schemaLocation,

			FormInformation: FormInformation{
				PreparedBy:   preparedBy,
				DatePrepared: time.Now().Format(DateFormat),
				ReportType:   "DYNAMIC",
			},

			SiteIdentification: SiteIdentification{
				SiteName:            m.Name,
				FourCharacterID:     m.Code,
				MonumentInscription: "",
				IersDOMESNumber:     monument.DomesNumber,
				CdpNumber:           "",
				MonumentDescription: monument.Type,
				HeightOfTheMonument: strconv.FormatFloat(-monument.GroundRelationship, 'g', -1, 64),
				MonumentFoundation:  monument.FoundationType,
				FoundationDepth:     strconv.FormatFloat(monument.FoundationDepth, 'f', 1, 64),
				MarkerDescription: func() string {
					switch monument.MarkType {
					case "Forced Centering":
						return "Forced Centering"
					default:
						return "unknown"
					}
				}(),
				DateInstalled:          m.Start.Format(DateTimeFormat),
				GeologicCharacteristic: "",
				BedrockType:            "",
				BedrockCondition:       "",
				FractureSpacing:        "",
				FaultZonesNearby:       "",
				DistanceActivity:       "",
				Notes:                  "",
			},
			SiteLocation: SiteLocation{
				/*
					City:          m.Place,
					State:         m.Region,
				*/
				Country: func(lat, lon float64) string {
					X, Y, _ := WGS842ITRF(lat, lon, 0.0)
					dist := float64(-1.0)
					country := "Unknown"
					for _, v := range countryList {
						x, y, _ := WGS842ITRF(v.lat, v.lon, 0.0)
						r := math.Sqrt((x-X)*(x-X) + (y-Y)*(y-Y))
						if dist < 0.0 || r < dist {
							country = v.name
							dist = r
						}
					}

					return country
				}(m.Latitude, m.Longitude),

				TectonicPlate: TectonicPlate(m.Latitude, m.Longitude),
				ApproximatePositionITRF: ApproximatePositionITRF{
					XCoordinateInMeters: strconv.FormatFloat(X, 'f', 1, 64),
					YCoordinateInMeters: strconv.FormatFloat(Y, 'f', 1, 64),
					ZCoordinateInMeters: strconv.FormatFloat(Z, 'f', 1, 64),
					LatitudeNorth:       strconv.FormatFloat(m.Latitude, 'g', -1, 64),
					LongitudeEast:       strconv.FormatFloat(m.Longitude, 'g', -1, 64),
					ElevationMEllips:    strconv.FormatFloat(m.Elevation, 'f', 1, 64),
				},
				Notes: "",
			},
			GnssReceivers:  receivers,
			GnssAntennas:   antennas,
			GnssMetSensors: metsensors,
			ContactAgency:  contactAgency,
			ResponsibleAgency: func() Agency {
				switch m.Network {
				case "LI":
					return responsibleAgency
				default:
					return Agency{
						MailingAddress: "\n",
					}
				}
			}(),
			MoreInformation: MoreInformation{
				PrimaryDataCenter:     primaryDatacentre,
				SecondaryDataCenter:   "",
				UrlForMoreInformation: urlForMoreInformation,
				HardCopyOnFile:        "",
				SiteMap:               "",
				SiteDiagram:           "",
				HorizonMask:           "",
				MonumentDescription:   "",
				SitePictures:          "",
				Notes:                 extraNotes + " " + m.Code,
				AntennaGraphicsWithDimensions: func() string {
					var graphs []string
					models := make(map[string]interface{})
					for _, a := range antennas {
						if _, ok := models[a.AntennaType]; ok {
							continue
						}
						if g, ok := antennaGraphs[a.AntennaType]; ok {
							b, err := hex.DecodeString(g)
							if err != nil {
								log.Printf("error: unable to decode antenna graph for: \"%s\"", a.AntennaType)
								continue
							}
							graphs = append(graphs, strings.Join([]string{a.AntennaType, string(b)}, "\n"))
						} else {
							log.Printf("warning: missing antenna graph for: \"%s\"", a.AntennaType)
						}
						models[a.AntennaType] = true
					}
					return strings.Join(graphs, "\n") + "\n"
				}(),
				InsertTextGraphicFromAntenna: "",
			},
		}

		s, err := x.Marshal()
		if err != nil {
			log.Fatalf("error: unable to marshal xml: %v", err)
		}

		xmlfile := filepath.Join(output, strings.ToLower(m.Code)+".xml")
		if err := os.MkdirAll(filepath.Dir(xmlfile), 0755); err != nil {
			log.Fatalf("error: unable to create dir: %v", err)
		}
		if err := ioutil.WriteFile(xmlfile, s, 0644); err != nil {
			log.Fatalf("error: unable to write file: %v", err)
		}

		logfile := filepath.Join(logs, strings.ToLower(m.Code)+".log")
		if err := os.MkdirAll(filepath.Dir(logfile), 0755); err != nil {
			log.Fatalf("error: unable to create logs dir: %v", err)
		}
		f, err := os.Create(logfile)
		if err != nil {
			log.Fatalf("error: unable to create log file: %v", err)
		}
		defer f.Close()

		if err := tmpl.Execute(f, x); err != nil {
			log.Fatalf("error: unable to write log file: %v", err)
		}
	}
}
Beispiel #18
0
func main() {

	var verbose bool
	flag.BoolVar(&verbose, "verbose", false, "make noise")

	var output string
	flag.StringVar(&output, "output", "output", "output directory")

	var network string
	flag.StringVar(&network, "network", "../../network", "base network directory")

	var install string
	flag.StringVar(&install, "install", "../../install", "base install directory")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "Build a RINEX configuration site XML file from delta meta information\n")
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "Usage:\n")
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "  %s [options]\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "Options:\n")
		fmt.Fprintf(os.Stderr, "\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
	}

	flag.Parse()

	var firmwareHistoryList meta.FirmwareHistoryList
	if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load firmware history: %v\n", err)
		os.Exit(-1)
	}

	firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory)
	for _, i := range firmwareHistoryList {
		if _, ok := firmwareHistory[i.Model]; !ok {
			firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory)
		}
		firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i)
	}

	for j, _ := range firmwareHistory {
		for k, _ := range firmwareHistory[j] {
			sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k]))
		}
	}

	var installedAntennaList meta.InstalledAntennaList
	if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load antenna installs: %v\n", err)
		os.Exit(-1)
	}

	installedAntenna := make(map[string][]meta.InstalledAntenna)
	for _, i := range installedAntennaList {
		installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i)
	}
	for i, _ := range installedAntenna {
		sort.Sort(meta.InstalledAntennaList(installedAntenna[i]))
	}

	var deployedReceiverList meta.DeployedReceiverList
	if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load receiver installs: %v\n", err)
		os.Exit(-1)
	}

	deployedReceivers := make(map[string][]meta.DeployedReceiver)
	for _, i := range deployedReceiverList {
		deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i)
	}
	for i, _ := range deployedReceivers {
		sort.Sort(meta.DeployedReceiverList(deployedReceivers[i]))
	}

	var installedRadomeList meta.InstalledRadomeList
	if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load radome installs: %v\n", err)
		os.Exit(-1)
	}

	installedRadomes := make(map[string][]meta.InstalledRadome)
	for _, i := range installedRadomeList {
		installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i)
	}
	for i, _ := range installedRadomes {
		sort.Sort(meta.InstalledRadomeList(installedRadomes[i]))
	}

	var markList meta.MarkList
	if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load mark list: %v\n", err)
		os.Exit(-1)
	}

	var sessionList meta.SessionList
	if err := meta.LoadList(filepath.Join(install, "sessions.csv"), &sessionList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load session list: %v\n", err)
		os.Exit(-1)
	}

	sessions := make(map[string][]meta.Session)
	for _, s := range sessionList {
		sessions[s.Mark] = append(sessions[s.Mark], s)
	}

	for _, m := range markList {

		if _, ok := sessions[m.Code]; !ok {
			continue
		}
		if _, ok := installedAntenna[m.Code]; !ok {
			continue
		}
		if _, ok := deployedReceivers[m.Code]; !ok {
			continue
		}

		var list []CGPSSessionXML
		for _, s := range sessions[m.Code] {
			for _, a := range installedAntenna[m.Code] {
				if a.Start.After(s.End) || a.End.Before(s.Start) {
					continue
				}
				if _, ok := IGSModels[a.Model]; !ok {
					fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s], skipping\n", m.Code, a.Model, a.Serial)
					continue
				}

				for _, r := range deployedReceivers[m.Code] {
					if r.Start.After(a.End) || r.End.Before(a.Start) {
						continue
					}
					if _, ok := IGSModels[r.Model]; !ok {
						fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s]", m.Code, r.Model, r.Serial)
						continue
					}

					radome := "NONE"
					if _, ok := installedRadomes[m.Code]; ok {
						for _, v := range installedRadomes[m.Code] {
							if v.Start.After(a.End) || v.Start.After(r.End) {
								continue
							}
							if v.End.Before(a.Start) || v.End.Before(r.Start) {
								continue
							}
							if _, ok := IGSModels[v.Model]; !ok {
								fmt.Fprintf(os.Stderr, "%s: no igs designation for: %s [%s]", m.Code, v.Model, v.Serial)
								continue
							}
							radome = IGSModels[v.Model]
						}
					}

					var firmware []FirmwareHistoryXML
					if _, ok := firmwareHistory[r.Model]; ok {
						if _, ok := firmwareHistory[r.Model][r.Serial]; ok {
							for i, _ := range firmwareHistory[r.Model][r.Serial] {
								v := firmwareHistory[r.Model][r.Serial][len(firmwareHistory[r.Model][r.Serial])-i-1]
								if v.End.Before(r.Start) || v.Start.After(r.End) {
									continue
								}
								firmware = append(firmware, FirmwareHistoryXML{
									StartTime: v.Start.Format(meta.DateTimeFormat),
									StopTime: func() string {
										if time.Now().After(v.End) {
											return v.End.Format(meta.DateTimeFormat)
										} else {
											return "open"
										}
									}(),
									Version: v.Version,
								})
							}
						}
					}

					list = append(list, CGPSSessionXML{
						StartTime: func() string {
							if r.Start.After(a.Start) {
								return r.Start.Format(meta.DateTimeFormat)
							} else if a.Start.After(s.Start) {
								return a.Start.Format(meta.DateTimeFormat)
							} else {
								return s.Start.Format(meta.DateTimeFormat)
							}
						}(),
						StopTime: func() string {
							if r.End.Before(a.End) {
								if time.Now().After(r.End) {
									return r.End.Format(meta.DateTimeFormat)
								} else {
									return "open"
								}
							} else if a.End.Before(s.End) {
								if time.Now().After(a.End) {
									return a.End.Format(meta.DateTimeFormat)
								} else {
									return "open"
								}
							} else {
								if time.Now().After(s.End) {
									return s.End.Format(meta.DateTimeFormat)
								} else {
									return "open"
								}
							}
						}(),
						Receiver: ReceiverXML{
							SerialNumber:      r.Serial,
							IGSDesignation:    IGSModels[r.Model],
							FirmwareHistories: firmware,
						},
						InstalledCGPSAntenna: InstalledCGPSAntennaXML{
							Height:      Number{Units: "m", Value: a.Vertical},
							OffsetNorth: Number{Units: "m", Value: a.North},
							OffsetEast:  Number{Units: "m", Value: a.East},
							Radome:      radome,
							CGPSAntenna: CGPSAntennaXML{
								SerialNumber:   a.Serial,
								IGSDesignation: IGSModels[a.Model],
							},
						},
						ObservationInterval: Number{
							Units: "s",
							Value: s.Interval.Seconds(),
						},
						Operator: OperatorXML{
							Name:   s.Operator,
							Agency: s.Agency,
						},
						Rinex: RinexXML{
							HeaderCommentName: s.HeaderComment,
							HeaderCommentText: func() string {
								if t, ok := HeaderComments[s.HeaderComment]; ok {
									return strings.Join(strings.Fields(t), " ")
								}
								return ""
							}(),
						},
						DataFormat: func() string {
							if f, ok := DataFormats[r.Model]; ok {
								return f
							}
							return "unknown"
						}(),
						DownloadNameFormat: func() DownloadNameFormatXML {
							if f, ok := DownloadNameFormats[r.Model]; ok {
								return f
							}
							return DownloadNameFormatXML{}
						}(),
					})
				}
			}
		}

		x := NewSiteXML(
			MarkXML{
				GeodeticCode: m.Code,
				//DomesNumber:  m.DomesNumber,
			},
			LocationXML{
				Latitude:  m.Latitude,
				Longitude: m.Longitude,
				Height:    m.Elevation,
				Datum:     m.Datum,
			},
			list,
		)

		s, err := x.Marshal()
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to marsh xml: %v\n", err)
			os.Exit(-1)
		}

		xmlfile := filepath.Join(output, m.Code+".xml")
		if err := os.MkdirAll(filepath.Dir(xmlfile), 0755); err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to create dir: %v\n", err)
			os.Exit(-1)
		}
		if err := ioutil.WriteFile(xmlfile, s, 0644); err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err)
			os.Exit(-1)
		}
	}
}
Beispiel #19
0
func TestStreams(t *testing.T) {

	var streams meta.StreamList

	t.Log("Load streams file")
	if err := meta.LoadList("../install/streams.csv", &streams); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < len(streams); i++ {
		for j := i + 1; j < len(streams); j++ {
			if streams[i].Station != streams[j].Station {
				continue
			}
			if streams[i].Location != streams[j].Location {
				continue
			}
			if streams[i].Start.After(streams[j].End) {
				continue
			}
			if streams[i].End.Before(streams[j].Start) {
				continue
			}
			if streams[i].SamplingRate != streams[j].SamplingRate {
				continue
			}
			t.Errorf("stream overlap: " + strings.Join([]string{
				streams[i].Station,
				streams[i].Location,
				streams[i].Start.String(),
				streams[i].End.String(),
			}, " "))
		}
	}

	stas := make(map[string]meta.Station)
	{
		var list meta.StationList
		t.Log("Load stations file")
		if err := meta.LoadList("../network/stations.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			stas[s.Code] = s
		}
	}

	sites := make(map[string]map[string]meta.Site)
	{
		var list meta.SiteList
		t.Log("Load sites file")
		if err := meta.LoadList("../network/sites.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			if _, ok := sites[s.Station]; !ok {
				sites[s.Station] = make(map[string]meta.Site)
			}
			sites[s.Station][s.Location] = s
		}
	}

	for _, s := range streams {
		if s.SamplingRate == 0 {
			t.Errorf("invalid stream sample rate: " + strings.Join([]string{
				s.Station,
				s.Location,
				s.Start.String(),
				s.End.String(),
			}, " "))
		}
	}

	for _, c := range streams {
		if _, ok := stas[c.Station]; !ok {
			t.Log("unknown stream station: " + c.Station)
		} else if s, ok := sites[c.Station]; !ok {
			t.Log("unknown stream station: " + c.Station)
		} else if _, ok := s[c.Location]; !ok {
			t.Log("unknown stream station/location: " + c.Station + "/" + c.Location)
		}
		if c.Start.After(c.End) {
			t.Log("stream span mismatch: " + strings.Join([]string{
				c.Station,
				c.Location,
				c.Start.String(),
				"after",
				c.End.String(),
			}, " "))
		}
	}

	var assets = make(map[string]meta.Asset)
	{
		var list meta.AssetList
		t.Log("Load recorders assets file")
		if err := meta.LoadList("../assets/recorders.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, l := range list {
			assets[l.Model+":::"+l.Serial] = l
		}
		t.Log("Load sensors assets file")
		if err := meta.LoadList("../assets/sensors.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			assets[s.Model+":::"+s.Serial] = s
		}
	}

	var recorders []meta.InstalledRecorder
	{
		var list meta.InstalledRecorderList
		t.Log("Load recorders file")
		if err := meta.LoadList("../install/recorders.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, r := range list {
			recorders = append(recorders, r)
		}
	}

	var missing []meta.Stream

	for _, r := range recorders {
		if a, ok := assets[r.Model+":::"+r.Serial]; !ok || a.Number == "" {
			continue
		}
		if r.End.Before(time.Now()) {
			continue
		}
		var handled bool
		for _, s := range streams {
			if s.Station != r.Station || r.Location != s.Location {
				continue
			}
			if r.Start.After(s.End) || r.End.Before(s.Start) {
				continue
			}
			handled = true
		}
		if !handled {
			for _, sps := range recorderSamplingRates {
				missing = append(missing, meta.Stream{
					Station:      r.Station,
					Location:     r.Location,
					SamplingRate: sps,
					Span: meta.Span{
						Start: r.Start,
						End:   r.End,
					},
				})
			}
			t.Errorf("no current stream defined for recorder: %s [%s/%s] %s %s", r.String(), r.Station, r.Location, r.Start, r.End)
		}
	}

	var sensors []meta.InstalledSensor
	{
		var list meta.InstalledSensorList
		t.Log("Load sensors file")
		if err := meta.LoadList("../install/sensors.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			sensors = append(sensors, s)
		}
	}

	for _, v := range sensors {
		if a, ok := assets[v.Model+":::"+v.Serial]; !ok || a.Number == "" {
			continue
		}
		if v.End.Before(time.Now()) {
			continue
		}
		var handled bool
		for _, s := range streams {
			if s.Station != v.Station || v.Location != s.Location {
				continue
			}
			if v.Start.After(s.End) || v.End.Before(s.Start) {
				continue
			}
			handled = true
		}
		if !handled {
			for _, sps := range sensorSamplingRates {
				missing = append(missing, meta.Stream{
					Station:      v.Station,
					Location:     v.Location,
					SamplingRate: sps,
					Span: meta.Span{
						Start: v.Start,
						End:   v.End,
					},
				})
			}
			t.Errorf("no current stream defined for sensor: %s [%s/%s] %s %s", v.String(), v.Station, v.Location, v.Start, v.End)
		}
	}

	if len(missing) > 0 {
		sort.Sort(meta.StreamList(missing))
		t.Log("\n" + string(meta.MarshalList(meta.StreamList(missing))))
	}

}
Beispiel #20
0
// TestMarksProto creates a Protobuf file of Marks.
func TestMarksProto(t *testing.T) {
	var networks meta.NetworkList

	if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
		t.Error(err)
	}

	var net = make(map[string]*delta.Network)

	for _, v := range networks {
		n := delta.Network{
			Code:        v.Code,
			External:    v.External,
			Description: v.Description,
			Restricted:  v.Restricted,
		}

		net[v.Code] = &n
	}

	var marks meta.MarkList
	if err := meta.LoadList("../network/marks.csv", &marks); err != nil {
		t.Error(err)
	}

	if len(marks) == 0 {
		t.Error("zero length mark list.")
	}

	var m delta.Marks

	m.Marks = make(map[string]*delta.Mark)

	for _, v := range marks {
		pt := delta.Point{
			Longitude: v.Longitude,
			Latitude:  v.Latitude,
			Elevation: v.Elevation,
			Datum:     v.Datum,
		}

		s := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}

		mk := delta.Mark{
			Code:    v.Code,
			Name:    v.Name,
			Network: net[v.Network],
			Point:   &pt,
			Span:    &s,
		}

		m.Marks[mk.Code] = &mk
	}

	b, err := proto.Marshal(&m)
	if err != nil {
		t.Error(err)
	}

	if err := os.MkdirAll(apiDir, 0777); err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(apiDir+"/marks.pb", b, 0644); err != nil {
		t.Error(err)
	}
}
Beispiel #21
0
func TestSessions(t *testing.T) {

	var sessions meta.SessionList

	t.Log("Load sessions file")
	if err := meta.LoadList("../install/sessions.csv", &sessions); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < len(sessions); i++ {
		for j := i + 1; j < len(sessions); j++ {
			if sessions[i].Mark != sessions[j].Mark {
				continue
			}
			if sessions[i].Model != sessions[j].Model {
				continue
			}
			if sessions[i].Interval != sessions[j].Interval {
				continue
			}
			if sessions[i].Start.After(sessions[j].End) {
				continue
			}
			if !sessions[i].End.After(sessions[j].Start) {
				continue
			}
			t.Errorf("session overlap: " + strings.Join([]string{
				sessions[i].Mark,
				sessions[i].Interval.String(),
				sessions[i].Start.String(),
				sessions[i].End.String(),
				sessions[j].Interval.String(),
				sessions[j].Start.String(),
				sessions[j].End.String(),
			}, " "))
		}
	}

	marks := make(map[string]meta.Mark)
	{
		var list meta.MarkList
		t.Log("Load marks file")
		if err := meta.LoadList("../network/marks.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, m := range list {
			marks[m.Code] = m
		}
	}

	for _, s := range sessions {
		if _, ok := marks[s.Mark]; !ok {
			t.Log("unknown session mark: " + s.Mark)
		}
		if s.Start.After(s.End) {
			t.Log("session span mismatch: " + strings.Join([]string{
				s.Mark,
				s.Interval.String(),
				s.Start.String(),
				"after",
				s.End.String(),
			}, " "))
		}
		switch s.SatelliteSystem {
		case "GPS":
		case "GPS+GLO":
		case "GPS+GLO+GAL+BDS+QZSS":
		default:
			t.Error("unknown satellite system: " + s.SatelliteSystem)
		}
	}
}
Beispiel #22
0
func TestCombined(t *testing.T) {

	var combined meta.InstalledSensorList
	t.Log("Load installed sensors file")
	{
		if err := meta.LoadList("../install/sensors.csv", &combined); err != nil {
			t.Fatal(err)
		}
	}

	var recorders meta.InstalledRecorderList
	t.Log("Load installed recorders file")
	{
		if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil {
			t.Fatal(err)
		}
	}

	for _, r := range recorders {
		combined = append(combined, meta.InstalledSensor{
			Install:  r.Install,
			Station:  r.Station,
			Location: r.Location,
		})
	}

	t.Log("Check for sensor/recorder installation location overlaps")
	{
		installs := make(map[string]meta.InstalledSensorList)
		for _, s := range combined {
			_, ok := installs[s.Station]
			if ok {
				installs[s.Station] = append(installs[s.Station], s)

			} else {
				installs[s.Station] = meta.InstalledSensorList{s}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Location != v[j].Location:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("sensor/recorder %s/%s at %-5s has location %-2s overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Station, v[i].Location, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

}
Beispiel #23
0
// TestStationsProto creates:
// Protobuf file of Stations
// JSON file of Stations
// GeoJSON of Site for each sensor type.
func TestStationsProto(t *testing.T) {
	var networks meta.NetworkList

	if err := meta.LoadList("../network/networks.csv", &networks); err != nil {
		t.Error(err)
	}

	var net = make(map[string]*delta.Network)

	for _, v := range networks {
		n := delta.Network{
			Code:        v.Code,
			External:    v.External,
			Description: v.Description,
			Restricted:  v.Restricted,
		}

		net[v.Code] = &n
	}

	var stations meta.StationList
	if err := meta.LoadList("../network/stations.csv", &stations); err != nil {
		t.Error(err)
	}

	if len(stations) == 0 {
		t.Error("zero length stations list.")
	}

	var s delta.Stations
	s.Stations = make(map[string]*delta.Station)

	for _, v := range stations {
		pt := delta.Point{
			Longitude: v.Longitude,
			Latitude:  v.Latitude,
			Elevation: v.Elevation,
			Datum:     v.Datum,
		}

		sp := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}

		st := delta.Station{
			Code:    v.Code,
			Name:    v.Name,
			Network: net[v.Network],
			Point:   &pt,
			Span:    &sp,
		}

		s.Stations[st.Code] = &st
	}

	var sites meta.SiteList
	if err := meta.LoadList("../network/sites.csv", &sites); err != nil {
		t.Error(err)
	}

	for _, v := range sites {
		pt := delta.Point{
			Longitude: v.Longitude,
			Latitude:  v.Latitude,
			Elevation: v.Elevation,
			Datum:     v.Datum,
		}
		sp := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		si := delta.Site{
			Location: v.Location,
			Survey:   v.Survey,
			Span:     &sp,
			Point:    &pt,
		}

		if _, ok := s.Stations[v.Station]; ok {
			if s.Stations[v.Station].Sites == nil {
				s.Stations[v.Station].Sites = make(map[string]*delta.Site)
			}
			s.Stations[v.Station].Sites[si.Location] = &si
		}
	}

	var installed meta.InstalledSensorList
	if err := meta.LoadList("../install/sensors.csv", &installed); err != nil {
		t.Fatal(err)
	}

	for _, v := range installed {
		st := resp.SensorModels[v.Model]

		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
			Type:   st.Type,
		}
		sp := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		o := delta.Orientation{
			Dip:     v.Dip,
			Azimuth: v.Azimuth,
		}
		off := delta.Offset{
			North:    v.North,
			East:     v.East,
			Vertical: v.Vertical,
		}
		sc := delta.Scale{
			Bias:   v.Bias,
			Factor: v.Factor,
		}
		is := delta.InstalledSensor{
			Equipment:   &e,
			Span:        &sp,
			Orientation: &o,
			Offset:      &off,
			Scale:       &sc,
		}

		if _, ok := s.Stations[v.Station]; ok {
			if _, ok := s.Stations[v.Station].Sites[v.Location]; ok {
				s.Stations[v.Station].Sites[v.Location].InstalledSensor = append(s.Stations[v.Station].Sites[v.Location].InstalledSensor, &is)
			}
		}
	}

	// strong motion recorders that are a sensor and a datalogger as a package.
	var recorders meta.InstalledRecorderList
	if err := meta.LoadList("../install/recorders.csv", &recorders); err != nil {
		t.Fatal(err)
	}

	for _, v := range recorders {
		st, ok := resp.SensorModels[v.InstalledSensor.Model]
		if !ok {
			// in resp/auto.go some models have " SENSOR" appended.
			st, ok = resp.SensorModels[v.InstalledSensor.Model+" SENSOR"]
		}

		e := delta.Equipment{
			Make:   v.Make,
			Model:  v.Model,
			Serial: v.Serial,
			Type:   st.Type,
		}
		sp := delta.Span{
			Start: v.Start.Unix(),
			End:   v.End.Unix(),
		}
		o := delta.Orientation{
			Dip:     v.Dip,
			Azimuth: v.Azimuth,
		}
		off := delta.Offset{
			North:    v.North,
			East:     v.East,
			Vertical: v.Vertical,
		}
		sc := delta.Scale{
			Bias:   v.Bias,
			Factor: v.Factor,
		}
		is := delta.InstalledSensor{
			Equipment:   &e,
			Span:        &sp,
			Orientation: &o,
			Offset:      &off,
			Scale:       &sc,
		}

		if _, ok := s.Stations[v.Station]; ok {
			if _, ok := s.Stations[v.Station].Sites[v.Location]; ok {
				s.Stations[v.Station].Sites[v.Location].InstalledSensor = append(s.Stations[v.Station].Sites[v.Location].InstalledSensor, &is)
			}
		}
	}

	// protobuf of all station information

	b, err := proto.Marshal(&s)
	if err != nil {
		t.Error(err)
	}

	if err := os.MkdirAll(apiDir, 0777); err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(apiDir+"/stations.pb", b, 0644); err != nil {
		t.Error(err)
	}

	// json of all station information

	b, err = json.Marshal(&s)
	if err != nil {
		t.Error(err)
	}

	if err := os.MkdirAll(apiDir, 0777); err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(apiDir+"/stations.json", b, 0644); err != nil {
		t.Error(err)
	}

	// GeoJSON files of site for each sensor type.

	var out = make(map[string]*bytes.Buffer)

	for _, v := range resp.SensorModels {
		if v.Type != "" {
			out[v.Type] = &bytes.Buffer{}
		}
	}

	for k := range out {
		out[k].WriteString(`{"type": "FeatureCollection","features": [`)
	}

	for _, station := range s.Stations {
		if station.GetSpan() == nil || station.GetPoint() == nil || station.GetNetwork() == nil || station.Network.Restricted {
			continue
		}
		if station.Network.External != "NZ" || station.Network.Code == "XX" {
			continue
		}

		for _, site := range station.Sites {
			for _, v := range site.InstalledSensor {
				_, ok := out[v.Equipment.Type]

				switch {
				case v.Equipment.Type == "":
					t.Logf("%s.%s no sensor model for %s, site not classified.",
						station.Code, site.Location, v.Equipment.Model)
				case ok:
					writeProps(station, site, v, out[v.Equipment.Type])
				case !ok:
					t.Logf("unclassified type %s", v.Equipment.Type)
				}
			}
		}
	}

	for k := range out {
		out[k].WriteString(`]}`)

		fn := strings.Replace(strings.ToLower(k), " ", "", -1)
		fn = apiDir + "/" + fn + ".geojson"

		if err := ioutil.WriteFile(fn, out[k].Bytes(), 0644); err != nil {
			t.Error(err)
		}
	}
}
Beispiel #24
0
func TestDataloggers(t *testing.T) {
	var dataloggers meta.DeployedDataloggerList

	t.Log("Load deployed dataloggers file")
	{
		if err := meta.LoadList("../install/dataloggers.csv", &dataloggers); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for datalogger installation place overlaps")
	{
		installs := make(map[string]meta.DeployedDataloggerList)
		for _, d := range dataloggers {
			_, ok := installs[d.Place]
			if ok {
				installs[d.Place] = append(installs[d.Place], d)

			} else {
				installs[d.Place] = meta.DeployedDataloggerList{d}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Place != v[j].Place:
					case v[i].Role != v[j].Role:
					//case v[i].Model != v[j].Model:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					//case v[i].End.Equal(v[j].Start):
					//case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("datalogger %s:[%s] at %-32s has place overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Place, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for datalogger installation equipment overlaps")
	{
		installs := make(map[string]meta.DeployedDataloggerList)
		for _, s := range dataloggers {
			_, ok := installs[s.Model]
			if ok {
				installs[s.Model] = append(installs[s.Model], s)

			} else {
				installs[s.Model] = meta.DeployedDataloggerList{s}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
						//		case v[i].End.Equal(v[j].Start):
						//		case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("datalogger %s:[%s] at %-32s has installation overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Place, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	var assets meta.AssetList
	t.Log("Load datalogger assets file")
	{
		if err := meta.LoadList("../assets/dataloggers.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for datalogger assets")
	{
		for _, r := range dataloggers {
			var found bool
			for _, a := range assets {
				if a.Model != r.Model {
					continue
				}
				if a.Serial != r.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find datalogger asset: %s [%s]", r.Model, r.Serial)
			}
		}
	}

}
Beispiel #25
0
func NewMetaDB(base string) (*MetaDB, error) {
	db, err := memdb.NewMemDB(NewSchema())
	if err != nil {
		return nil, err
	}

	var networks = []struct {
		table string
		path  string
		list  meta.List
	}{
		{"gauge", "network/gauges.csv", &meta.GaugeList{}},
		{"constituent", "network/constituents.csv", &meta.ConstituentList{}},
		{"mark", "network/marks.csv", &meta.MarkList{}},
		{"monument", "network/monuments.csv", &meta.MonumentList{}},
		{"mount", "network/mounts.csv", &meta.MountList{}},
		{"network", "network/networks.csv", &meta.NetworkList{}},
		{"site", "network/sites.csv", &meta.SiteList{}},
		{"station", "network/stations.csv", &meta.StationList{}},
	}

	txn := db.Txn(true)
	for _, list := range networks {
		if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) {
			continue
		}

		switch list.list.(type) {
		case *meta.ConstituentList:
			var input meta.ConstituentList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.GaugeList:
			var input meta.GaugeList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.MarkList:
			var input meta.MarkList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.MonumentList:
			var input meta.MonumentList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.MountList:
			var input meta.MountList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.NetworkList:
			var input meta.NetworkList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.SiteList:
			var input meta.SiteList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.StationList:
			var input meta.StationList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		default:
			return nil, fmt.Errorf("invalid type found for: %s", list.path)
		}
	}
	txn.Commit()

	var assets = []struct {
		table string
		path  string
	}{
		{"antenna", "assets/antennas.csv"},
		{"camera", "assets/cameras.csv"},
		{"datalogger", "assets/dataloggers.csv"},
		{"metsensor", "assets/metsensors.csv"},
		{"radome", "assets/radomes.csv"},
		{"receiver", "assets/receivers.csv"},
		{"recorder", "assets/recorders.csv"},
		{"sensor", "assets/sensors.csv"},
	}

	txn = db.Txn(true)
	for _, list := range assets {
		if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) {
			continue
		}

		var input meta.AssetList
		if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
			return nil, err
		}
		for _, i := range input {
			if err := txn.Insert("asset", i); err != nil {
				return nil, err
			}
		}

	}
	txn.Commit()

	var installs = []struct {
		table string
		path  string
		list  meta.List
	}{
		{"antenna", "install/antennas.csv", &meta.InstalledAntennaList{}},
		{"camera", "install/cameras.csv", &meta.InstalledCameraList{}},
		{"datalogger", "install/dataloggers.csv", &meta.DeployedDataloggerList{}},
		{"metsensor", "install/metsensors.csv", &meta.InstalledMetSensorList{}},
		{"radome", "install/radomes.csv", &meta.InstalledRadomeList{}},
		{"receiver", "install/receivers.csv", &meta.DeployedReceiverList{}},
		{"recorder", "install/recorders.csv", &meta.InstalledRecorderList{}},
		{"sensor", "install/sensors.csv", &meta.InstalledSensorList{}},
		{"connection", "install/connections.csv", &meta.ConnectionList{}},
		{"firmware", "install/firmware.csv", &meta.FirmwareHistoryList{}},
		{"session", "install/sessions.csv", &meta.SessionList{}},
		{"stream", "install/streams.csv", &meta.StreamList{}},
	}

	txn = db.Txn(true)
	for _, list := range installs {
		if _, err := os.Stat(filepath.Join(base, list.path)); os.IsNotExist(err) {
			continue
		}

		switch list.list.(type) {
		case *meta.InstalledAntennaList:
			var input meta.InstalledAntennaList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.InstalledCameraList:
			var input meta.InstalledCameraList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.DeployedDataloggerList:
			var input meta.DeployedDataloggerList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.InstalledMetSensorList:
			var input meta.InstalledMetSensorList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.InstalledRadomeList:
			var input meta.InstalledRadomeList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.DeployedReceiverList:
			var input meta.DeployedReceiverList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.InstalledRecorderList:
			var input meta.InstalledRecorderList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.InstalledSensorList:
			var input meta.InstalledSensorList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.ConnectionList:
			var input meta.ConnectionList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.FirmwareHistoryList:
			var input meta.FirmwareHistoryList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.SessionList:
			var input meta.SessionList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		case *meta.StreamList:
			var input meta.StreamList
			if err := meta.LoadList(filepath.Join(base, list.path), &input); err != nil {
				return nil, err
			}
			for _, i := range input {
				if err := txn.Insert(list.table, i); err != nil {
					return nil, err
				}
			}
		default:
			return nil, fmt.Errorf("invalid type found for: %s", list.path)
		}
	}
	txn.Commit()

	return &MetaDB{MemDB: db}, nil
}
Beispiel #26
0
func TestConsistency(t *testing.T) {

	files := map[string]struct {
		f string
		l meta.List
	}{
		"connections":  {f: "../install/connections.csv", l: &meta.ConnectionList{}},
		"cameras":      {f: "../install/cameras.csv", l: &meta.InstalledCameraList{}},
		"dataloggers":  {f: "../install/dataloggers.csv", l: &meta.DeployedDataloggerList{}},
		"metsensors":   {f: "../install/metsensors.csv", l: &meta.InstalledMetSensorList{}},
		"radomes":      {f: "../install/radomes.csv", l: &meta.InstalledRadomeList{}},
		"receivers":    {f: "../install/receivers.csv", l: &meta.DeployedReceiverList{}},
		"recorders":    {f: "../install/recorders.csv", l: &meta.InstalledRecorderList{}},
		"sensors":      {f: "../install/sensors.csv", l: &meta.InstalledSensorList{}},
		"firmware":     {f: "../install/firmware.csv", l: &meta.FirmwareHistoryList{}},
		"streams":      {f: "../install/streams.csv", l: &meta.StreamList{}},
		"networks":     {f: "../network/networks.csv", l: &meta.NetworkList{}},
		"stations":     {f: "../network/stations.csv", l: &meta.StationList{}},
		"sites":        {f: "../network/sites.csv", l: &meta.SiteList{}},
		"marks":        {f: "../network/marks.csv", l: &meta.MarkList{}},
		"mounts":       {f: "../network/mounts.csv", l: &meta.MountList{}},
		"gauges":       {f: "../network/gauges.csv", l: &meta.GaugeList{}},
		"constituents": {f: "../network/constituents.csv", l: &meta.ConstituentList{}},
	}

	for k, v := range files {
		if err := meta.LoadList(v.f, v.l); err != nil {
			t.Fatal(err)
		}

		sort.Sort(v.l)

		raw, err := ioutil.ReadFile(v.f)
		if err != nil {
			t.Fatal(err)
		}

		var buf bytes.Buffer
		if err := csv.NewWriter(&buf).WriteAll(meta.EncodeList(v.l)); err != nil {
			t.Fatal(err)
		}

		if string(raw) != buf.String() {
			t.Error(k + ": **** csv file mismatch **** : " + v.f)

			file, err := ioutil.TempFile(os.TempDir(), "tst")
			if err != nil {
				t.Fatal(err)
			}
			defer os.Remove(file.Name())
			file.Write(buf.Bytes())

			cmd := exec.Command("diff", "-c", v.f, file.Name())
			stdout, err := cmd.StdoutPipe()
			if err != nil {
				t.Fatal(err)
			}
			err = cmd.Start()
			if err != nil {
				t.Fatal(err)
			}
			defer cmd.Wait()
			diff, err := ioutil.ReadAll(stdout)
			if err != nil {
				t.Fatal(err)
			}
			t.Error(string(diff))
		}
	}
}
Beispiel #27
0
func NewMeta(network, install string) (*Meta, error) {
	networkMap := make(map[string]meta.Network)

	var n meta.NetworkList
	if err := meta.LoadList(filepath.Join(network, "networks.csv"), &n); err != nil {
		return nil, err
	}

	for _, v := range n {
		networkMap[v.Code] = v
	}

	stationMap := make(map[string]meta.Station)

	var stations meta.StationList
	if err := meta.LoadList(filepath.Join(network, "stations.csv"), &stations); err != nil {
		return nil, err
	}

	for _, v := range stations {
		stationMap[v.Code] = v
	}

	connectionMap := make(map[string]meta.ConnectionList)

	var connections meta.ConnectionList
	if err := meta.LoadList(filepath.Join(install, "connections.csv"), &connections); err != nil {
		return nil, err
	}

	for _, c := range connections {
		if _, ok := connectionMap[c.Station]; ok {
			connectionMap[c.Station] = append(connectionMap[c.Station], c)
		} else {
			connectionMap[c.Station] = meta.ConnectionList{c}
		}
	}

	var recorders meta.InstalledRecorderList
	if err := meta.LoadList(filepath.Join(install, "recorders.csv"), &recorders); err != nil {
		return nil, err
	}
	for _, r := range recorders {
		c := meta.Connection{
			Station:  r.Station,
			Location: r.Location,
			Span: meta.Span{
				Start: r.Start,
				End:   r.End,
			},
			Place: r.Station,
			Role:  r.Location,
		}
		if _, ok := connectionMap[c.Station]; ok {
			connectionMap[c.Station] = append(connectionMap[c.Station], c)
		} else {
			connectionMap[c.Station] = meta.ConnectionList{c}
		}
	}

	siteMap := make(map[string]map[string]meta.Site)

	var locations meta.SiteList
	if err := meta.LoadList(filepath.Join(network, "sites.csv"), &locations); err != nil {
		return nil, err
	}

	for _, l := range locations {
		if _, ok := siteMap[l.Station]; !ok {
			siteMap[l.Station] = make(map[string]meta.Site)
		}
		siteMap[l.Station][l.Location] = l
	}

	streamMap := make(map[string]map[string]meta.StreamList)

	var streams meta.StreamList
	if err := meta.LoadList(filepath.Join(install, "streams.csv"), &streams); err != nil {
		return nil, err
	}

	for _, s := range streams {
		if _, ok := streamMap[s.Station]; !ok {
			streamMap[s.Station] = make(map[string]meta.StreamList)
		}

		if _, ok := streamMap[s.Station][s.Location]; ok {
			streamMap[s.Station][s.Location] = append(streamMap[s.Station][s.Location], s)
		} else {
			streamMap[s.Station][s.Location] = meta.StreamList{s}
		}
	}

	sensorInstalls := make(map[string]meta.InstalledSensorList)

	// build sensor installation details
	var sensors meta.InstalledSensorList
	if err := meta.LoadList(filepath.Join(install, "sensors.csv"), &sensors); err != nil {
		return nil, err
	}
	for _, s := range sensors {
		if _, ok := sensorInstalls[s.Station]; ok {
			sensorInstalls[s.Station] = append(sensorInstalls[s.Station], s)
		} else {
			sensorInstalls[s.Station] = meta.InstalledSensorList{s}
		}
	}

	for _, r := range recorders {
		if _, ok := sensorInstalls[r.Station]; ok {
			sensorInstalls[r.Station] = append(sensorInstalls[r.Station], r.InstalledSensor)
		} else {
			sensorInstalls[r.Station] = meta.InstalledSensorList{r.InstalledSensor}
		}
	}

	for i, _ := range sensorInstalls {
		sort.Sort(sensorInstalls[i])
	}

	dataloggerDeploys := make(map[string]meta.DeployedDataloggerList)

	// where the dataloggers were deployed
	var loggers meta.DeployedDataloggerList
	if err := meta.LoadList(filepath.Join(install, "dataloggers.csv"), &loggers); err != nil {
		return nil, err
	}
	for _, d := range loggers {
		if _, ok := dataloggerDeploys[d.Place]; ok {
			dataloggerDeploys[d.Place] = append(dataloggerDeploys[d.Place], d)
		} else {
			dataloggerDeploys[d.Place] = meta.DeployedDataloggerList{d}
		}
	}

	for _, r := range recorders {
		d := meta.DeployedDatalogger{
			Install: meta.Install{
				Equipment: meta.Equipment{
					Make:   r.Make,
					Model:  r.DataloggerModel,
					Serial: r.Serial,
				},
				Span: meta.Span{
					Start: r.Start,
					End:   r.End,
				},
			},
			Place: r.Station,
			Role:  r.Location,
		}
		if _, ok := dataloggerDeploys[d.Place]; ok {
			dataloggerDeploys[d.Place] = append(dataloggerDeploys[d.Place], d)
		} else {
			dataloggerDeploys[d.Place] = meta.DeployedDataloggerList{d}
		}
	}

	// sort each datalogger deployment
	for i, _ := range dataloggerDeploys {
		sort.Sort(dataloggerDeploys[i])
	}

	return &Meta{
		Networks:    networkMap,
		Stations:    stationMap,
		Connections: connectionMap,
		Sites:       siteMap,
		Streams:     streamMap,
		Installs:    sensorInstalls,
		Deploys:     dataloggerDeploys,
	}, nil
}
Beispiel #28
0
func TestCameras(t *testing.T) {

	var cameras meta.InstalledCameraList
	t.Log("Load deployed cameras file")
	{
		if err := meta.LoadList("../install/cameras.csv", &cameras); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for cameras installation equipment overlaps")
	{
		installs := make(map[string]meta.InstalledCameraList)
		for _, s := range cameras {
			_, ok := installs[s.Model]
			if ok {
				installs[s.Model] = append(installs[s.Model], s)

			} else {
				installs[s.Model] = meta.InstalledCameraList{s}
			}
		}

		var keys []string
		for k, _ := range installs {
			keys = append(keys, k)
		}
		sort.Strings(keys)

		for _, k := range keys {
			v := installs[k]

			for i, n := 0, len(v); i < n; i++ {
				for j := i + 1; j < n; j++ {
					switch {
					case v[i].Serial != v[j].Serial:
					case v[i].End.Before(v[j].Start):
					case v[i].Start.After(v[j].End):
					case v[i].End.Equal(v[j].Start):
					case v[i].Start.Equal(v[j].End):
					default:
						t.Errorf("cameras %s at %-5s has mount %s overlap between %s and %s",
							v[i].Model, v[i].Serial, v[i].Mount, v[i].Start.Format(meta.DateTimeFormat), v[i].End.Format(meta.DateTimeFormat))
					}
				}
			}
		}
	}

	t.Log("Check for missing camera mounts")
	{
		var mounts meta.MountList

		if err := meta.LoadList("../network/mounts.csv", &mounts); err != nil {
			t.Fatal(err)
		}

		keys := make(map[string]interface{})

		for _, m := range mounts {
			keys[m.Code] = true
		}

		for _, c := range cameras {
			if _, ok := keys[c.Mount]; ok {
				continue
			}
			t.Errorf("unable to find camera mount %-5s", c.Mount)
		}
	}

	var assets meta.AssetList
	t.Log("Load camera assets file")
	{
		if err := meta.LoadList("../assets/cameras.csv", &assets); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Check for camera assets")
	{
		for _, r := range cameras {
			var found bool
			for _, a := range assets {
				if a.Model != r.Model {
					continue
				}
				if a.Serial != r.Serial {
					continue
				}
				found = true
			}
			if !found {
				t.Errorf("unable to find camera asset: %s [%s]", r.Model, r.Serial)
			}
		}
	}

}
Beispiel #29
0
func TestConnections(t *testing.T) {

	var connections meta.ConnectionList

	t.Log("Load connections file")
	if err := meta.LoadList("../install/connections.csv", &connections); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < len(connections); i++ {
		for j := i + 1; j < len(connections); j++ {
			if connections[i].Station != connections[j].Station {
				continue
			}
			if connections[i].Location != connections[j].Location {
				continue
			}
			if connections[i].Start.After(connections[j].End) {
				continue
			}
			if connections[i].End.Before(connections[j].Start) {
				continue
			}
			t.Errorf("connection overlap: " + strings.Join([]string{
				connections[i].Station,
				connections[i].Location,
				connections[i].Start.String(),
				connections[i].End.String(),
			}, " "))
		}
	}

	stas := make(map[string]meta.Station)
	{
		var list meta.StationList
		t.Log("Load stations file")
		if err := meta.LoadList("../network/stations.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			stas[s.Code] = s
		}
	}

	sites := make(map[string]map[string]meta.Site)
	{
		var list meta.SiteList
		t.Log("Load sites file")
		if err := meta.LoadList("../network/sites.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, s := range list {
			if _, ok := sites[s.Station]; !ok {
				sites[s.Station] = make(map[string]meta.Site)
			}
			sites[s.Station][s.Location] = s
		}
	}

	for _, c := range connections {
		if _, ok := stas[c.Station]; !ok {
			t.Log("unknown connection station: " + c.Station)
		} else if s, ok := sites[c.Station]; !ok {
			t.Log("unknown connection station: " + c.Station)
		} else if _, ok := s[c.Location]; !ok {
			t.Log("unknown connection station/location: " + c.Station + "/" + c.Location)
		}
		if c.Start.After(c.End) {
			t.Log("connection span mismatch: " + strings.Join([]string{
				c.Station,
				c.Location,
				c.Start.String(),
				"after",
				c.End.String(),
			}, " "))
		}
	}

	places := make(map[string]string)
	{
		var list meta.DeployedDataloggerList
		t.Log("Load installed dataloggers file")
		if err := meta.LoadList("../install/dataloggers.csv", &list); err != nil {
			t.Fatal(err)
		}
		for _, d := range list {
			if d.Role != "" {
				places[d.Place+"/"+d.Role] = d.Place
			} else {
				places[d.Place] = d.Place
			}
		}
	}

	for _, c := range connections {
		if c.Role != "" {
			if _, ok := places[c.Place+"/"+c.Role]; !ok {
				t.Log("warning: unknown datalogger place/role: " + c.Place + "/" + c.Role)
			}
		} else {
			if _, ok := places[c.Place]; !ok {
				t.Log("warning: unknown datalogger place: " + c.Place)
			}
		}
	}

	/*
		Station Code,Location Code,Datalogger Place,Datalogger Role,Pre Amp,Gain,Start Date,End Date
		A11,10,Alfredton,Short Period,false,0,1993-02-01T01:23:00Z,1993-02-21T08:48:00Z
		ABAZ,10,Whangaparaoa Navy Base,,false,0,2008-10-13T04:00:00Z,9999-01-01T00:00:00Z
		AC1A,10,Wards Pass #1,,false,0,2001-09-12T03:00:01Z,2002-01-22T22:00:00Z
		AC2A,10,Wards Pass #2,,false,0,2001-09-13T02:00:01Z,2002-01-22T22:00:00Z
		AC3A,10,Acheron #3,,false,0,2001-07-31T23:00:00Z,2002-01-22T23:00:00Z
		AC4A,10,Acheron #4,Short Period,false,0,2001-08-01T01:00:00Z,2002-01-22T23:00:00Z
		AC5A,10,Wards Pass #5,,false,0,2001-08-01T04:00:00Z,2001-11-27T23:00:00Z
		AGA,10,Angora Road,Short Period,false,0,1990-02-20T22:33:00Z,1992-04-07T17:01:00Z
		AHAA,10,Ahaura,Short Period,false,0,1995-11-19T17:31:00Z,1995-12-12T01:28:00Z
	*/

	/*
	 */
}