func TestLocators(t *testing.T) {
google, err := geolocate.New(geolocate.Google, googleAPIKey)
if err != nil {
t.Fatal(err)
}
mozilla, err := geolocate.New(geolocate.Mozilla, "test")
if err != nil {
t.Fatal(err)
}
yandex, err := geolocate.New(geolocate.Yandex, yandexAPIKey)
if err != nil {
t.Fatal(err)
}
mongodb, err := mongo.Connect("mongodb://localhost/watch")
if err != nil {
t.Fatal(err)
}
defer mongodb.Close()
lbs, err := lbs.InitDB(mongodb)
if err != nil {
t.Fatal(err)
}
for _, s := range testData {
req, err := ParseLBS("gsm", s)
if err != nil {
t.Fatal(err)
}
resp, err := google.Get(*req)
if err != nil {
fmt.Printf("%8s error: %v\n", "Google", err)
t.Error(err)
} else {
fmt.Printf("%8s [%.7f,%.7f] %.2f\n", "Google", resp.Location.Lon, resp.Location.Lat, resp.Accuracy)
}
resp, err = yandex.Get(*req)
if err != nil {
fmt.Printf("%8s error: %v\n", "Yandex", err)
t.Error(err)
} else {
fmt.Printf("%8s [%.7f,%.7f] %.2f\n", "Yandex", resp.Location.Lon, resp.Location.Lat, resp.Accuracy)
}
resp, err = mozilla.Get(*req)
if err != nil {
fmt.Printf("%8s error: %v\n", "Mozilla", err)
t.Error(err)
} else {
fmt.Printf("%8s [%.7f,%.7f] %.2f\n", "Mozilla", resp.Location.Lon, resp.Location.Lat, resp.Accuracy)
}
resp, err = lbs.Get(*req)
if err != nil {
fmt.Printf("%8s error: %v\n", "Internal", err)
t.Error(err)
}
fmt.Printf("%8s [%.7f,%.7f] %.2f\n", "Internal", resp.Location.Lon, resp.Location.Lat, resp.Accuracy)
fmt.Println("-----------------------------------------")
}
}
func main() {
log.SetOutput(os.Stdout)
log.SetFlags(log.Ltime)
mongourl := flag.String("mongo", "mongodb://localhost/watch", "mongoDB connection URL")
radiofilter := flag.String("radio", "gsm", "filter for radio (comma separated)")
countryfilter := flag.String("country", "250", "filter for country (comma separated)")
minSamples := flag.Int64("minsample", 0, "filter for min samples count")
flag.Usage = func() {
fmt.Fprint(os.Stderr, "Import LBS database data\n")
fmt.Fprintf(os.Stderr, "%s [-params] datafile.csv\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
if flag.NArg() != 1 {
flag.Usage()
return
}
filename := flag.Arg(0)
// устанавливаем соединение с сервером MongoDB
log.Printf("Connecting to MongoDB %q...", *mongourl)
mdb, err := mongo.Connect(*mongourl)
if err != nil {
log.Printf("Error connecting to MongoDB: %v", err)
return
}
defer mdb.Close()
db, err := lbs.InitDB(mdb)
if err != nil {
log.Printf("Error initializing indexes: %v", err)
return
}
coll := db.GetCollection(lbs.CollectionName)
defer db.FreeCollection(coll)
bulk := coll.Bulk()
bulk.Unordered()
// разбираем фильтры и формируем соответствующие справочники
var (
filterRadio = make(map[string]bool)
filterCountry = make(map[uint16]bool)
)
for _, radio := range strings.Split(*radiofilter, ",") {
filterRadio[strings.ToLower(strings.TrimSpace(radio))] = true
}
for _, country := range strings.Split(*countryfilter, ",") {
mcc, err := strconv.ParseUint(country, 10, 16)
if err != nil {
continue
}
filterCountry[uint16(mcc)] = true
}
if len(filterRadio) > 0 || len(filterCountry) > 0 {
log.Printf("Filters country - %q, radio - %q",
strings.Join(strings.Split(*countryfilter, ","), ", "),
strings.Join(strings.Split(*radiofilter, ","), ", "))
}
log.Printf("Reading data from CSV %q...", filename)
file, err := os.Open(filename)
if err != nil {
log.Printf("Error opening CSV file: %v", err)
return
}
defer file.Close()
var counter, lines uint64 // счетчики
r := csv.NewReader(file)
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
log.Printf("Error parsing CSV file: %v", err)
return
}
lines++
if lines == 1 {
r.FieldsPerRecord = len(record) // устанавливаем количество полей
continue // пропускаем первую строку с заголовком в CSV-файле
}
fmt.Fprintf(os.Stderr, "\r* find %8d | skipped %8d records ", counter, lines-1-counter)
radio := strings.ToLower(record[0])
if len(filterRadio) > 0 && !filterRadio[radio] {
continue // игнорируем записи с неподдерживаемым типом радио
}
samples, err := strconv.ParseInt(record[9], 10, 32)
if err != nil {
log.Printf("[%d] bad Samples: %s", lines, record[9])
continue
}
if samples < *minSamples {
continue // не импортируем данные с маленьким количеством подтверждений
}
mcc, err := strconv.ParseUint(record[1], 10, 16)
if err != nil {
log.Printf("[%d] bad MCC: %s", lines, record[1])
continue
}
if len(filterCountry) > 0 && !filterCountry[uint16(mcc)] {
continue // игнорируем записи с неподдерживаемым типом радио
}
mnc, err := strconv.ParseUint(record[2], 10, 16)
if err != nil {
log.Printf("[%d] bad MNC: %s", lines, record[2])
continue
}
area, err := strconv.ParseUint(record[3], 10, 16)
if err != nil {
log.Printf("[%d] bad Area: %s", lines, record[3])
continue
}
cell, err := strconv.ParseUint(record[4], 10, 32)
if err != nil {
log.Printf("[%d] bad Cell: %s", lines, record[4])
continue
}
lon, err := strconv.ParseFloat(record[6], 64)
if err != nil {
log.Printf("[%d] bad longitude:", lines, record[6])
continue
}
lat, err := strconv.ParseFloat(record[7], 64)
if err != nil {
log.Printf("[%d] bad latitude:", lines, record[7])
continue
}
distance, err := strconv.ParseFloat(record[8], 64)
if err != nil {
log.Printf("[%d] bad range:", lines, record[8])
continue
}
key := lbs.Key{
RadioType: radio,
MobileCountryCode: uint16(mcc),
MobileNetworkCode: uint16(mnc),
LocationAreaCode: uint16(area),
CellId: uint32(cell),
}
data := lbs.Data{
Location: geo.NewPoint(lon, lat),
Accuracy: distance,
}
// created, err := strconv.ParseInt(record[11], 10, 64)
// if err != nil {
// log.Printf("[%d] bad Created: %s", lines, record[11])
// continue
// }
// updated, err := strconv.ParseInt(record[12], 10, 64)
// if err != nil {
// log.Printf("[%d] bad Updated: %s", lines, record[12])
// continue
// }
bulk.Upsert(key, bson.M{"$set": data})
counter++
}
fmt.Fprintln(os.Stderr, "")
if counter == 0 {
log.Println("No record for import. Exit...")
return
}
// если это не обновление, то подчищаем старые (не обновленные) данные
if !strings.Contains(filename, "diff") {
log.Println("Deleting old data...")
deleteResult, err := coll.RemoveAll(nil)
if err != nil {
log.Printf("MongoDB deleting old data error: %v", err)
return
}
if deleteResult.Removed > 0 {
log.Printf("Deleted %d records", deleteResult.Removed)
}
}
log.Printf("Bulk importing to MongoDB [%d records]...", counter)
bulkResult, err := bulk.Run()
if err != nil {
log.Printf("MongoDB bulk insert error: %v", err)
return
}
if bulkResult.Modified > 0 {
log.Printf("Modified %d records", bulkResult.Modified)
}
total, err := coll.Count()
if err != nil {
log.Printf("MongoDB total counting error: %v", err)
return
}
log.Printf("Total unique records in DB: %d", total)
}
func subscribe(mdb *mongo.DB, nc *nats.Conn) error {
nce, err := nats.NewEncodedConn(nc, nats.JSON_ENCODER)
if err != nil {
return err
}
// nce.Subscribe("*", func(subj, reply string, data []byte) {
// log.Printf("DEBUG: %q [%q]\n%s", subj, reply, string(data))
// })
lbs, err := lbs.InitDB(mdb)
if err != nil {
return err
}
if lbs.Records() == 0 {
log.Warn("LBS DB is empty!")
}
lbsGoogle, err := geolocate.New(geolocate.Google, googleToken)
if err != nil {
return err
}
nce.Subscribe(serviceNameLBS, func(_, reply string, req geolocate.Request) {
resp, err := lbsGoogle.Get(req)
logger := log.WithFields(log.Fields{"request": req, "response": resp})
if err != nil {
logger.WithError(err).Error("LBS Google error")
}
if err := nce.Publish(reply, resp); err != nil {
logger.WithError(err).Error("LBS Google response error")
} else {
logger.Debug("LBS")
}
_, err = lbs.Get(req) // оставил для сохранения в базу запроса.
if err != nil {
logger.WithError(err).Error("LBS Internal response error")
}
})
ubloxCache, err := ublox.InitCache(mdb, ubloxToken)
if err != nil {
return err
}
profile := ublox.DefaultProfile
nce.Subscribe(serviceNameUblox, func(_, reply string, point geo.Point) {
data, err := ubloxCache.Get(point, profile)
logger := log.WithFields(log.Fields{"request": point, "response length": len(data)})
if err != nil {
logger.WithError(err).Error("UBLOX error")
}
if err := nce.Publish(reply, data); err != nil {
logger.WithError(err).Error("UBLOX response error")
} else {
logger.Debug("UBLOX")
}
})
usersDB, err := users.InitDB(mdb)
if err != nil {
return err
}
// уникальный идентификатор группы пока для примера задан явно
// groupID := users.SampleGroupID
groupID := usersDB.GetSampleGroupID()
nce.Subscribe(serviceNameIMEI, func(_, reply, data string) {
group, err := usersDB.GetGroup(groupID)
logger := log.WithFields(log.Fields{"request": data, "response": group})
if err != nil {
logger.WithError(err).Error("IMEI error")
}
if err := nce.Publish(reply, group); err != nil {
logger.WithError(err).Error("IMEI response error")
} else {
logger.Debug("IMEI")
}
})
tracksDB, err := tracks.InitDB(mdb)
if err != nil {
return err
}
nce.Subscribe(serviceNameTracks, func(tracks []tracks.TrackData) {
logger := log.WithField("request", tracks)
if err := tracksDB.Add(tracks...); err != nil {
logger.WithError(err).Error("TRACKS error")
} else {
logger.Debug("TRACKS")
}
})
sensorsDB, err := sensors.InitDB(mdb)
if err != nil {
return err
}
nce.Subscribe(serviceNameSensors, func(sensors []sensors.SensorData) {
logger := log.WithField("request", sensors)
if err := sensorsDB.Add(sensors...); err != nil {
logger.WithError(err).Error("SENSORS error")
} else {
logger.Debug("SENSORS")
}
})
var pairs pairing.Pairs
nce.Subscribe(serviceNamePairing, func(_, reply, deviceID string) {
key := pairs.Generate(deviceID)
logger := log.WithFields(log.Fields{"request": deviceID, "response": key})
if err := nce.Publish(reply, key); err != nil {
logger.WithError(err).Error("PAIR error")
} else {
logger.Debug("PAIR")
}
})
nce.Subscribe(serviceNamePairingKey, func(_, reply, key string) {
newDeviceID := pairs.GetDeviceID(key)
logger := log.WithFields(log.Fields{"request": key, "response": newDeviceID})
if err := nce.Publish(reply, newDeviceID); err != nil {
logger.WithError(err).Error("PAIR KEY error")
} else {
logger.Debug("PAIR KEY")
}
})
return nil
}