func scanAlternateNames(
db *bolt.DB, filename string, locales []string,
) (int, error) {
file, err := os.Open(filename)
if err != nil {
return 0, err
}
defer file.Close()
reader := bufio.NewReader(file)
scanner := bufio.NewScanner(reader)
cityNamesCount := 0
countriesTranslations := make(map[int]map[string]string)
err = db.Update(func(tx *bolt.Tx) error {
cityNamesBucket := tx.Bucket(ds.CityNamesBucketName)
countriesBucket := tx.Bucket(ds.CountriesBucketName)
for scanner.Scan() {
nameData := strings.Split(scanner.Text(), "\t")
if isSupportedLocale(nameData[2], locales) || nameData[4] == "1" {
city, _ := ds.FindCity(db, nameData[1], false)
if city != nil {
addCityToIndex(
cityNamesBucket, strconv.Itoa(city.ID), nameData[3],
nameData[2], city.Population,
)
cityNamesCount++
} else if nameData[2] != "en" {
country, _ := ds.FindCountry(db, nameData[1])
if country != nil {
if countriesTranslations[country.ID] == nil {
countriesTranslations[country.ID] = make(map[string]string)
}
if countriesTranslations[country.ID][nameData[2]] == "" {
countriesTranslations[country.ID][nameData[2]] = nameData[3]
}
}
}
}
}
for id, translations := range countriesTranslations {
var values []string
for locale, name := range translations {
values = append(values, locale+"|"+name)
}
addTranslationsToCountry(countriesBucket, id, values)
}
return err
})
return cityNamesCount, err
}
func TestScanAlternateNames(t *testing.T) {
Convey("Test scan alternate names", t, func() {
db := h.CreateDB(t)
ds.CreateCitiesBucket(db)
ds.CreateCityNamesBucket(db)
ds.CreateCountriesBucket(db)
h.PutToBucket(t, db, ds.CitiesBucketName, "1", "Montreal\t\t\t\t\t")
h.PutToBucket(t, db, ds.CitiesBucketName, "2", "Moscow\t\t\t\t\t")
h.PutToBucket(t, db, ds.CountriesBucketName, "3", "DE\tGermany\ten|Germany")
locales := []string{"de", "ru"}
Convey("When alternate names file exists", func() {
filename := h.CreateTempfile(
t,
"10\t1\tfr\tMontréal\t\t\t\t\n11\t2\tde\tMoskau\t\t\t\t\n12\t2\tru\tМосква\t\t\t\t13\t9\tde\tMünchen\t\t\t\t\n"+
"14\t3\tde\tDeutschland\t\t\t\t\n15\t3\ten\tWest Germany\t\t\t\t\n16\t3\tit\tGermania\t\t\t\t",
)
count, err := scanAlternateNames(db, filename, locales)
country, _ := ds.FindCountry(db, "3")
Convey("Returns number of scanned records", func() {
So(count, ShouldEqual, 2)
})
Convey("When the locale is supported", func() {
Convey("Stores the record if the city exists", func() {
actual := h.ReadFromBucket(t, db, ds.CityNamesBucketName, "moskau")
So(actual, ShouldEqual, "Moskau\t2\tde\t0")
})
Convey("Doesn't store the record if the city doesn't exist", func() {
actual := h.ReadFromBucket(t, db, ds.CityNamesBucketName, "münchen")
So(actual, ShouldEqual, "")
})
Convey("Adds translations for countries", func() {
So(country.Translations["de"], ShouldEqual, "Deutschland")
})
Convey("Doesn't override en names for countries", func() {
So(country.Translations["en"], ShouldEqual, "Germany")
})
})
Convey("When the locale is not supported", func() {
Convey("Doesn't store the record", func() {
actual := h.ReadFromBucket(t, db, ds.CityNamesBucketName, "montréal")
So(actual, ShouldEqual, "")
})
Convey("Doesn't add translations for countries", func() {
So(country.Translations["it"], ShouldEqual, "")
})
})
Convey("Returns no error", func() {
So(err, ShouldBeNil)
})
})
Convey("When alternate names file does not exist", func() {
count, err := scanAlternateNames(db, "fake.txt", locales)
Convey("Returns a zero number of scanned records", func() {
So(count, ShouldEqual, 0)
})
Convey("Returns an error", func() {
So(err, ShouldNotBeNil)
})
})
})
}
func TestUtils(t *testing.T) {
db := h.CreateDB(t)
ds.CreateCityNamesBucket(db)
ds.CreateCountriesBucket(db)
Convey("Test prepare country bytes", t, func() {
Convey("When the data is correct", func() {
data := []string{
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10",
"11", "12", "13", "14", "15", "16", "17", "18",
}
expected := []byte("0\t4\ten|4")
actual, err := prepareCountryBytes(data)
Convey("Joins the array in the right order with tabs", func() {
So(actual, ShouldResemble, expected)
})
Convey("Returns no error", func() {
So(err, ShouldBeNil)
})
})
Convey("When the data is incorrect", func() {
data := []string{"yolo"}
actual, err := prepareCountryBytes(data)
Convey("Returns an empty bytes array", func() {
var bytes []byte
So(actual, ShouldResemble, bytes)
})
Convey("Returns an error", func() {
So(err, ShouldNotBeNil)
})
})
})
Convey("Test add translations to country", t, func() {
translations := []string{"de|Deutschland", "ru|Германия"}
countryAttrs := []string{"DE", "Germany", "en|Germany"}
countryString := strings.Join(countryAttrs, "\t")
h.PutToBucket(t, db, ds.CountriesBucketName, "1", countryString)
err := db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(ds.CountriesBucketName)
return addTranslationsToCountry(b, 1, translations)
})
country, err := ds.FindCountry(db, "1")
Convey("Does not modify country data", func() {
So(country.Code, ShouldEqual, countryAttrs[0])
So(country.Name, ShouldEqual, countryAttrs[1])
})
Convey("Keeps old translations", func() {
So(country.Translations["en"], ShouldEqual, "Germany")
})
Convey("Adds new translations", func() {
So(country.Translations["de"], ShouldEqual, "Deutschland")
So(country.Translations["ru"], ShouldEqual, "Германия")
})
Convey("Returns no error", func() {
So(err, ShouldBeNil)
})
})
Convey("Test prepare city bytes", t, func() {
Convey("When the data is correct", func() {
data := []string{
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10",
"11", "12", "13", "14", "15", "16", "17", "18",
}
expected := []byte("1\t8\t14\t4\t5\t17")
actual, err := prepareCityBytes(data)
Convey("Joins the array in the right order with tabs", func() {
So(actual, ShouldResemble, expected)
})
Convey("Returns no error", func() {
So(err, ShouldBeNil)
})
})
Convey("When the data is incorrect", func() {
data := []string{"yolo"}
actual, err := prepareCityBytes(data)
Convey("Returns an empty bytes array", func() {
var bytes []byte
So(actual, ShouldResemble, bytes)
})
Convey("Returns an error", func() {
So(err, ShouldNotBeNil)
})
})
})
Convey("Test add city to index", t, func() {
Convey("If no record for the key exist yet", func() {
err := db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(ds.CityNamesBucketName)
return addCityToIndex(b, "1", "Berlin", "en", 2000000)
})
Convey("Puts the city name string to the bucket", func() {
actual := h.ReadFromBucket(t, db, ds.CityNamesBucketName, "berlin")
So(actual, ShouldEqual, "Berlin\t1\ten\t2000000")
})
Convey("Not return an error", func() {
So(err, ShouldBeNil)
})
})
Convey("If a record for the key exists", func() {
existing := "Moscow\t1\ten\t12000000"
h.PutToBucket(t, db, ds.CityNamesBucketName, "moscow", existing)
err := db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(ds.CityNamesBucketName)
return addCityToIndex(b, "2", "Moscow", "en", 20000)
})
Convey("Does not overwrites the existing entry", func() {
actual := h.ReadFromBucket(t, db, ds.CityNamesBucketName, "moscow")
So(actual, ShouldEqual, existing)
})
Convey("Adds city id as postfix for a new entry key", func() {
actual := h.ReadFromBucket(t, db, ds.CityNamesBucketName, "moscow|2")
So(actual, ShouldEqual, "Moscow\t2\ten\t20000")
})
Convey("Not return an error", func() {
So(err, ShouldBeNil)
})
})
})
Convey("Test is supported locale", t, func() {
locales := []string{"ru", "en", "de"}
Convey("Returns true for supported locales", func() {
for _, locale := range locales {
So(isSupportedLocale(locale, locales), ShouldBeTrue)
}
})
Convey("Returns false for unsupported locales", func() {
So(isSupportedLocale("jp", locales), ShouldBeFalse)
})
})
}
