// Locate returns the point and uncertainty associated with a location as // interpreted by the geolocation service. Uncertainty indicates the maximum // uncertainty (in meters) accepted for the point (with 0 any uncertainty // will be accepted). func (s *service) Locate(l *geography.Location, locality string, uncertainty uint) (geography.Georeference, error) { ls, err := s.List(l, locality, uncertainty) if err != nil { return geography.Georeference{Point: geography.InvalidPoint()}, err } if len(ls) == 1 { return ls[0], nil } if len(ls) == 0 { return geography.Georeference{Point: geography.InvalidPoint()}, geography.ErrNotInDB } u := uncertainty if u == 0 { u = 200000 } // set a mid point var sLon, sLat float64 unc := uint(0) for _, p := range ls { sLon += p.Point.Lon sLat += p.Point.Lat if unc < p.Uncertainty { unc = p.Uncertainty } } den := float64(len(ls)) lon, lat := sLon/den, sLat/den if !(geography.IsLon(lon) && geography.IsLat(lat)) { return geography.Georeference{Point: geography.InvalidPoint()}, geography.ErrAmbiguous } max := uint(0) for _, p := range ls { d := p.Point.Distance(lon, lat) if d > max { max = d if (d + unc) > u { break } } } p := ls[0] p.Uncertainty = max + unc if (p.Uncertainty > u) || (!p.IsValid()) { return geography.Georeference{Point: geography.InvalidPoint()}, geography.ErrAmbiguous } p.Point = geography.Point{lon, lat} return p, nil }
func (o *occurrence) copy() *jdh.Specimen { cat := strings.TrimSpace(o.InstitutionCode) cat += ":" + strings.TrimSpace(o.CollectionCode) cat += ":" + strings.TrimSpace(o.CatalogNumber) t, _ := time.Parse("2006-01-02T15:04:05.000-0700", o.OccurrenceDate) spe := &jdh.Specimen{ Id: strconv.FormatInt(o.Key, 10), Taxon: strconv.FormatInt(o.TaxonKey, 10), Basis: getBasis(o.BasisOfRecord), Dataset: strings.TrimSpace(o.DatasetKey), Catalog: cat, Determiner: strings.Join(strings.Fields(o.IdentifierName), " "), Collector: strings.Join(strings.Fields(o.CollectorName), " "), Date: t, Geography: geography.Location{ Country: geography.GetCountry(o.CountryCode), State: strings.Join(strings.Fields(o.StateProvince), " "), County: strings.Join(strings.Fields(o.County), " "), }, Locality: strings.Join(strings.Fields(o.Locality), " "), Comment: strings.TrimSpace(o.FieldNotes + "\n" + o.OccurrenceRemarks), } if (len(spe.Locality) == 0) && (len(o.VerbatimLocality) > 0) { spe.Locality = strings.Join(strings.Fields(o.VerbatimLocality), " ") } lon, lat := float64(360), float64(360) if o.DecimalLongitude != 0 { lon = o.DecimalLongitude } if o.DecimalLatitude != 0 { lat = o.DecimalLatitude } if geography.IsLon(lon) && geography.IsLat(lat) { spe.Georef.Point = geography.Point{Lon: lon, Lat: lat} spe.Georef.Source = strings.Join(strings.Fields(o.GeoreferenceSources), " ") } else { spe.Georef = geography.InvalidGeoref() } return spe }
func spGrefProc(c *cmdapp.Command, tax *jdh.Taxon, gzt geography.Gazetter) { defer func() { l := getTaxDesc(c, localDB, tax.Id, true) spGrefNav(c, l, gzt) l = getTaxDesc(c, localDB, tax.Id, false) spGrefNav(c, l, gzt) }() if len(tax.Id) == 0 { return } vals := new(jdh.Values) vals.Add(jdh.SpeTaxon, tax.Id) if !addFlag { vals.Add(jdh.SpeGeoref, "true") } l := speList(c, localDB, vals) defer l.Close() for { spe := &jdh.Specimen{} if err := l.Scan(spe); err != nil { if err == io.EOF { break } fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } if len(spe.Georef.Validation) > 0 { continue } if !spe.Geography.IsValid() { fmt.Fprintf(os.Stdout, "%s: location without valid country\n", spe.Id) continue } u := uint(uncertFlag) if u == 0 { if u = spe.Georef.Uncertainty; u == 0 { // 200 km is the maximum validation u = 200000 } } if !spe.Georef.IsValid() { if !addFlag { fmt.Fprintf(os.Stdout, "%s: invalid georeference\n", spe.Id) continue } p, err := gzt.Locate(&spe.Geography, spe.Locality, uint(uncertFlag)) if err != nil { fmt.Fprintf(os.Stdout, "%s: unable to add: %v\n", spe.Id, err) if verboseFlag { if err == geography.ErrAmbiguous { pts, err := gzt.List(&spe.Geography, spe.Locality, uint(uncertFlag)) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } for _, p := range pts { fmt.Fprintf(os.Stderr, "\t%.5f %.5f\t%d\n", p.Point.Lon, p.Point.Lat, p.Uncertainty) } } } continue } vals := new(jdh.Values) vals.Add(jdh.KeyId, spe.Id) vals.Add(jdh.GeoLonLat, strconv.FormatFloat(p.Point.Lon, 'g', -1, 64)+","+strconv.FormatFloat(p.Point.Lat, 'g', -1, 64)) vals.Add(jdh.GeoUncertainty, strconv.FormatInt(int64(p.Uncertainty), 10)) vals.Add(jdh.GeoSource, p.Source) vals.Add(jdh.GeoValidation, p.Validation) localDB.Exec(jdh.Set, jdh.Specimens, vals) continue } pts, err := gzt.List(&spe.Geography, spe.Locality, u) if err != nil { fmt.Fprintf(os.Stdout, "%s: %v\n", spe.Id, err) continue } if len(pts) == 0 { fmt.Fprintf(os.Stdout, "%s: location not found\n", spe.Id) continue } lon, lat := spe.Georef.Point.Lon, spe.Georef.Point.Lat val := false gr := geography.Georeference{ Point: geography.InvalidPoint(), Uncertainty: geography.EarthRadius * 10, // a distance large enough } for _, p := range pts { d := p.Point.Distance(lon, lat) if d <= u { val = true if (d + p.Uncertainty) < gr.Uncertainty { gr = p gr.Uncertainty += d } } } if val { vals := new(jdh.Values) vals.Add(jdh.KeyId, spe.Id) if spe.Georef.Uncertainty == 0 { vals.Add(jdh.GeoUncertainty, strconv.FormatInt(int64(gr.Uncertainty), 10)) } vals.Add(jdh.GeoValidation, gr.Validation) localDB.Exec(jdh.Set, jdh.Specimens, vals) continue } if !corrFlag { fmt.Fprintf(os.Stdout, "%s: location not found\n", spe.Id) if verboseFlag { fmt.Fprintf(os.Stderr, "\t%.5f %.5f\t\t[current georeference]\n", lon, lat) for _, p := range pts { fmt.Fprintf(os.Stderr, "\t%.5f %.5f\t%d\t%d\n", p.Point.Lon, p.Point.Lat, p.Uncertainty, p.Point.Distance(lon, lat)) } } continue } gr = geography.Georeference{ Point: geography.InvalidPoint(), Uncertainty: geography.EarthRadius * 10, // a distance large enough } val = false for _, p := range pts { // invert lon-lat lt, ln := lon, lat if geography.IsLon(ln) && geography.IsLat(lt) { d := p.Point.Distance(lon, lat) if d <= u { val = true gr = p gr.Point = geography.Point{Lon: ln, Lat: lt} gr.Uncertainty += d break } } // lon with wrong sign ln, lt = -lon, lat if geography.IsLon(ln) && geography.IsLat(lt) { d := p.Point.Distance(lon, lat) if d <= u { val = true gr = p gr.Point = geography.Point{Lon: ln, Lat: lt} gr.Uncertainty += d break } } // lat with wrong sing ln, lt = lon, -lat if geography.IsLon(ln) && geography.IsLat(lt) { d := p.Point.Distance(lon, lat) if d <= u { val = true gr = p gr.Point = geography.Point{Lon: ln, Lat: lt} gr.Uncertainty += d break } } // invert lon-lat, wrong sings lt, ln = -lon, -lat if geography.IsLon(ln) && geography.IsLat(lt) { d := p.Point.Distance(lon, lat) if d <= u { val = true gr = p gr.Point = geography.Point{Lon: ln, Lat: lt} gr.Uncertainty += d break } } // invert lon-lat, lon with wrong sing lt, ln = lon, -lat if geography.IsLon(ln) && geography.IsLat(lt) { d := p.Point.Distance(lon, lat) if d <= u { val = true gr = p gr.Point = geography.Point{Lon: ln, Lat: lt} gr.Uncertainty += d break } } // invert lon-lat, lat with wrong sing lt, ln = -lon, lat if geography.IsLon(ln) && geography.IsLat(lt) { d := p.Point.Distance(lon, lat) if d <= u { val = true gr = p gr.Point = geography.Point{Lon: ln, Lat: lt} gr.Uncertainty += d break } } } if val { vals := new(jdh.Values) vals.Add(jdh.KeyId, spe.Id) vals.Add(jdh.GeoLonLat, strconv.FormatFloat(gr.Point.Lon, 'g', -1, 64)+","+strconv.FormatFloat(gr.Point.Lat, 'g', -1, 64)) vals.Add(jdh.GeoUncertainty, strconv.FormatInt(int64(gr.Uncertainty), 10)) vals.Add(jdh.GeoSource, gr.Source) vals.Add(jdh.GeoValidation, gr.Validation) localDB.Exec(jdh.Set, jdh.Specimens, vals) continue } fmt.Fprintf(os.Stdout, "%s: not corrected\n", spe.Id) if verboseFlag { fmt.Fprintf(os.Stderr, "\t%.5f %.5f\t\t[current georeference]\n", lon, lat) for _, p := range pts { fmt.Fprintf(os.Stderr, "\t%.5f %.5f\t%d\t%d\n", p.Point.Lon, p.Point.Lat, p.Uncertainty, p.Point.Distance(lon, lat)) } } } }
func spInNdm(c *cmdapp.Command, fname, parent string, rank jdh.Rank, set *jdh.Dataset) { var in *bufio.Reader if len(fname) > 0 { f, err := os.Open(fname) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) return } defer f.Close() in = bufio.NewReader(f) } else { in = bufio.NewReader(os.Stdin) } var tok string var err error //read header var transpose, nocommas bool xneg, yneg := float64(1), float64(1) for { tok, err = readString(in) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } if tok == "xydata" { break } switch tok { case "ynegative": yneg = -1 case "xnegative": xneg = -1 case "transpose", "longlat": transpose = true case "nocommas": nocommas = true case "latlong": transpose = false } } tok, err = readString(in) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } for { if err == io.EOF { break } if (tok == ";") || (tok == "groups") || (tok == "map") { break } if tok != "sp" { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr("expecting 'sp' found: "+tok)) os.Exit(1) } txNum := "" txNum, err = readString(in) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } var r rune r, err = peekNext(in) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } var txname string if r == '[' { // feed the first rune in.ReadRune() nm, err := readBlock(in, ']') if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } fld := strings.Fields(nm) if len(fld) > 1 { // skips the optional filling lst := []rune(fld[len(fld)-1]) if unicode.IsNumber(lst[0]) { fld = fld[:len(fld)-1] } } txname = strings.Join(fld, " ") } // skips specimens for anonymous taxons if len(txname) == 0 { if verboseFlag { fmt.Fprintf(os.Stdout, "WARNING:\t%s\t\tUnnamed taxon\n", txNum) } tok, err = skipNdmTaxon(c, in) continue } mult, id := spInSearchNmdTaxon(c, txname, parent, txNum, rank) if mult { tok, err = skipNdmTaxon(c, in) continue } if len(id) == 0 { pId := parent // we know that the first name of a species is the genus. if rank == jdh.Species { tn := strings.Fields(txname) if len(tn) > 1 { if p := taxInDB(c, localDB, tn[0], parent, jdh.Genus, true); p != nil { pId = p.Id } else { par := &jdh.Taxon{ Name: tn[0], IsValid: true, Parent: parent, Rank: jdh.Genus, } pId, err = localDB.Exec(jdh.Add, jdh.Taxonomy, par) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) tok, err = skipNdmTaxon(c, in) continue } if verboseFlag { fmt.Fprintf(os.Stdout, "%s %s\n", pId, par.Name) } } } } tax := &jdh.Taxon{ Name: txname, IsValid: true, Parent: pId, Rank: rank, } id, err = localDB.Exec(jdh.Add, jdh.Taxonomy, tax) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) tok, err = skipNdmTaxon(c, in) continue } if verboseFlag { fmt.Fprintf(os.Stdout, "%s %s\n", id, tax.Name) } } else if skipFlag { tok, err = skipNdmTaxon(c, in) continue } // read the specimens speOk := 0 for { tok, err = readString(in) if err != nil { if err == io.EOF { break } fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } if (tok == "sp") || (tok == ";") || (tok == "groups") || (tok == "map") { break } var v1, v2 float64 if nocommas { val1 := tok tok, err = readString(in) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) os.Exit(1) } val2 := tok v1, err = strconv.ParseFloat(val1, 64) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } v2, err = strconv.ParseFloat(val2, 64) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } } else { v := strings.Split(tok, ",") if len(v) < 2 { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr("expecting more than one comma separtated values")) continue } v1, err = strconv.ParseFloat(v[0], 64) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } v2, err = strconv.ParseFloat(v[1], 64) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } } lon, lat := float64(360), float64(360) if transpose { lon = v1 * xneg lat = v2 * yneg } else { lon = v2 * xneg lat = v1 * yneg } if geography.IsLon(lon) && geography.IsLat(lat) { spe := &jdh.Specimen{ Taxon: id, Dataset: set.Id, } spe.Georef.Point = geography.Point{Lon: lon, Lat: lat} _, err = localDB.Exec(jdh.Add, jdh.Specimens, spe) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } speOk++ } else { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(fmt.Sprintf("invalid coordinates %.5f %.5f", v1, v2))) continue } } if verboseFlag { fmt.Fprintf(os.Stdout, "%s\t%s\t%s\t%d\tspecs added\n", txNum, txname, id, speOk) } } }
func spInTxt(c *cmdapp.Command, fname, parent string, rank jdh.Rank, set *jdh.Dataset) { var in *bufio.Reader if len(fname) > 0 { f, err := os.Open(fname) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) return } defer f.Close() in = bufio.NewReader(f) } else { in = bufio.NewReader(os.Stdin) } for { ln, err := readLine(in) if err != nil { break } if len(ln) < 3 { continue } id := ln[0] lon, err := strconv.ParseFloat(ln[1], 64) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } lat, err := strconv.ParseFloat(ln[2], 64) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } cat := "" if len(ln) > 3 { cat = ln[3] if sp := specimen(c, localDB, cat); len(sp.Id) > 0 { continue } } if ((!geography.IsLon(lon)) || (!geography.IsLat(lat))) && (len(cat) == 0) { continue } sId := set.Id if len(ln) > 4 { sId = ln[4] if ds := dataset(c, localDB, sId); len(ds.Id) > 0 { sId = ds.Id } else { sId = "" } } spe := &jdh.Specimen{ Taxon: id, Catalog: cat, Dataset: sId, } if geography.IsLon(lon) && geography.IsLat(lat) { spe.Georef.Point = geography.Point{Lon: lon, Lat: lat} } else { if len(cat) == 0 { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(fmt.Sprintf("invalid coordinates %.5f %.5f", lon, lat))) continue } spe.Georef = geography.InvalidGeoref() } _, err = localDB.Exec(jdh.Add, jdh.Specimens, spe) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", c.ErrStr(err)) continue } } }
// Set sets a value of an specimen in the database. func (s *specimens) set(vals []jdh.KeyValue) error { id := "" for _, kv := range vals { if len(kv.Value) == 0 { continue } if kv.Key == jdh.KeyId { id = kv.Value[0] break } } if len(id) == 0 { return errors.New("specimen without identification") } sp, ok := s.ids[id] if !ok { return nil } spe := sp.data for _, kv := range vals { switch kv.Key { case jdh.SpeBasis: v := jdh.UnknownBasis if len(kv.Value) > 0 { v = jdh.GetBasisOfRecord(strings.TrimSpace(kv.Value[0])) } if spe.Basis == v { continue } spe.Basis = v case jdh.SpeCatalog: v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } if spe.Catalog == v { continue } if len(v) > 0 { if _, ok := s.ids[v]; ok { return fmt.Errorf("specimen catalog code %s already in use", kv.Value) } } if len(spe.Catalog) > 0 { delete(s.ids, spe.Catalog) } spe.Catalog = v if len(v) > 0 { s.ids[v] = sp } case jdh.SpeCollector: v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Collector == v { continue } spe.Collector = v case jdh.SpeDataset: v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } if spe.Dataset == v { continue } if len(v) > 0 { if !s.db.d.isInDB(v) { continue } } spe.Dataset = v case jdh.SpeDate: if len(kv.Value) > 0 { v := strings.TrimSpace(kv.Value[0]) t, err := time.Parse(jdh.Iso8601, v) if err != nil { return err } if spe.Date.Equal(t) { continue } spe.Date = t break } if spe.Date.IsZero() { continue } spe.Date = time.Time{} case jdh.SpeDeterminer: v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Determiner == v { continue } spe.Determiner = v case jdh.SpeLocality: v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Locality == v { continue } spe.Locality = v case jdh.SpeTaxon: v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } if len(v) == 0 { continue } if spe.Taxon == v { continue } tax, ok := s.taxId[v] if !ok { if !s.db.t.isInDB(v) { continue } tax = &speTaxon{ id: v, specs: list.New(), } tax.elem = s.taxLs.PushBack(tax) s.taxId[tax.id] = tax } oldtax := sp.taxon oldtax.specs.Remove(sp.elem) sp.elem = tax.specs.PushBack(sp) sp.taxon = tax sp.data.Taxon = tax.id if oldtax.specs.Len() == 0 { oldtax.specs = nil s.taxLs.Remove(oldtax.elem) oldtax.elem = nil delete(s.taxId, oldtax.id) } case jdh.GeoCountry: v := geography.Country("") if len(kv.Value) > 0 { v = geography.GetCountry(strings.Join(strings.Fields(kv.Value[0]), " ")) } if spe.Geography.Country == v { continue } spe.Geography.Country = v case jdh.GeoCounty: v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Geography.County == v { continue } spe.Geography.County = v case jdh.GeoLonLat: v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } if len(v) == 0 { if !spe.Georef.IsValid() { continue } spe.Georef = geography.InvalidGeoref() break } coor := strings.Split(v, ",") if len(coor) != 2 { return errors.New("invalid geographic coordinate values") } lon, err := strconv.ParseFloat(coor[0], 64) if err != nil { return err } lat, err := strconv.ParseFloat(coor[1], 64) if err != nil { return err } if (lon == 0) || (lon == 1) || (lat == 0) || (lat == 1) { return errors.New("invalid geographic coordinate values") } if (!geography.IsLon(lon)) || (!geography.IsLat(lat)) { return errors.New("invalid geographic coordinate values") } spe.Georef.Point = geography.Point{Lon: lon, Lat: lat} case jdh.GeoSource: if !spe.Georef.IsValid() { continue } v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Georef.Source == v { continue } spe.Georef.Source = v case jdh.GeoState: v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Geography.State == v { continue } spe.Geography.State = v case jdh.GeoUncertainty: if !spe.Georef.IsValid() { continue } v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } un64, err := strconv.ParseUint(v, 10, 0) if err != nil { return err } un := uint(un64) if un == spe.Georef.Uncertainty { continue } spe.Georef.Uncertainty = un case jdh.GeoValidation: if !spe.Georef.IsValid() { continue } v := "" if len(kv.Value) > 0 { v = strings.Join(strings.Fields(kv.Value[0]), " ") } if spe.Georef.Validation == v { continue } spe.Georef.Validation = v case jdh.KeyComment: v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } if spe.Comment == v { continue } spe.Comment = v case jdh.KeyExtern: ok := false for _, v := range kv.Value { v = strings.TrimSpace(v) if len(v) == 0 { continue } serv, ext, err := jdh.ParseExtern(v) if err != nil { return err } if len(ext) == 0 { if !s.delExtern(sp, serv) { continue } ok = true continue } if s.addExtern(sp, v) != nil { continue } ok = true } if !ok { continue } case jdh.KeyReference: v := "" if len(kv.Value) > 0 { v = strings.TrimSpace(kv.Value[0]) } if spe.Reference == v { continue } spe.Reference = v default: continue } s.changed = true } return nil }