func TestParents(t *testing.T) { testCases := []struct { tag, parent string }{ {"af", "und"}, {"en", "und"}, {"en-001", "en"}, {"en-AU", "en-001"}, {"en-US", "en"}, {"en-US-u-va-posix", "en-US"}, {"ca-ES-valencia", "ca-ES"}, } for _, tc := range testCases { tag, ok := language.CompactIndex(language.MustParse(tc.tag)) if !ok { t.Fatalf("Could not get index of flag %s", tc.tag) } want, ok := language.CompactIndex(language.MustParse(tc.parent)) if !ok { t.Fatalf("Could not get index of parent %s of tag %s", tc.parent, tc.tag) } if got := int(Parent[tag]); got != want { t.Errorf("Parent[%s] = %d; want %d (%s)", tc.tag, got, want, tc.parent) } } }
func BenchmarkPluralComplexCases(b *testing.B) { p := &cardinalData ar, _ := language.CompactIndex(language.Arabic) lv, _ := language.CompactIndex(language.Latvian) for i := 0; i < b.N; i++ { matchPlural(p, lv, 0, 19, 2) // 0.19 matchPlural(p, lv, 11, 0, 3) // 11.000 matchPlural(p, lv, 100, 123, 4) // 0.1230 matchPlural(p, ar, 0, 0, 0) // 0 matchPlural(p, ar, 110, 0, 0) // 110 matchPlural(p, ar, 99, 99, 2) // 99.99 } }
func BenchmarkPluralSimpleCases(b *testing.B) { p := &cardinalData en, _ := language.CompactIndex(language.English) zh, _ := language.CompactIndex(language.Chinese) for i := 0; i < b.N; i++ { matchPlural(p, en, 0, 0, 0) // 0 matchPlural(p, en, 1, 0, 0) // 1 matchPlural(p, en, 2, 12, 3) // 2.120 matchPlural(p, zh, 0, 0, 0) // 0 matchPlural(p, zh, 1, 0, 0) // 1 matchPlural(p, zh, 2, 12, 3) // 2.120 } }
// Format implements fmt.Formatter. It accepts format.State for // language-specific rendering. func (v Value) Format(s fmt.State, verb rune) { var lang int if state, ok := s.(format.State); ok { lang, _ = language.CompactIndex(state.Language()) } // Get the options. Use DefaultFormat if not present. opt := v.format if opt == nil { opt = defaultFormat } cur := v.currency if cur.index == 0 { cur = opt.currency } // TODO: use pattern. io.WriteString(s, opt.symbol(lang, cur)) if v.amount != nil { s.Write(space) // TODO: apply currency-specific rounding scale, _ := opt.kind.Rounding(cur) if _, ok := s.Precision(); !ok { fmt.Fprintf(s, "%.*f", scale, v.amount) } else { fmt.Fprint(s, v.amount) } } }
func formatForLang(t language.Tag, index []byte) *Format { for ; ; t = t.Parent() { if x, ok := language.CompactIndex(t); ok { return &formats[index[x]] } } }
// InfoFromTag returns a Info for the given language tag. func InfoFromTag(t language.Tag) Info { for { if index, ok := language.CompactIndex(t); ok { return InfoFromLangID(index, t.TypeForKey("nu")) } t = t.Parent() } }
func main() { r := gen.OpenCLDRCoreZip() defer r.Close() d := &cldr.Decoder{} data, err := d.DecodeZip(r) if err != nil { log.Fatalf("DecodeZip: %v", err) } w := gen.NewCodeWriter() defer w.WriteGoFile("tables.go", "internal") // Create parents table. parents := make([]uint16, language.NumCompactTags) for _, loc := range data.Locales() { tag := language.MustParse(loc) index, ok := language.CompactIndex(tag) if !ok { continue } parentIndex := 0 // und for p := tag.Parent(); p != language.Und; p = p.Parent() { if x, ok := language.CompactIndex(p); ok { parentIndex = x break } } parents[index] = uint16(parentIndex) } w.WriteComment(` Parent maps a compact index of a tag to the compact index of the parent of this tag.`) w.WriteVar("Parent", parents) }
func testPlurals(t *testing.T, p *pluralRules, testCases []pluralTest) { for _, tc := range testCases { for _, loc := range strings.Split(tc.locales, " ") { langIndex, _ := language.CompactIndex(language.MustParse(loc)) // Test integers for _, s := range tc.integer { a := strings.Split(s, "~") from := parseUint(t, a[0]) to := from if len(a) > 1 { to = parseUint(t, a[1]) } for n := from; n <= to; n++ { if f := matchPlural(p, langIndex, n, 0, 0); f != tc.form { t.Errorf("%s:int(%d) = %v; want %v", loc, n, f, tc.form) } } } // Test decimals for _, s := range tc.decimal { a := strings.Split(s, "~") from, scale := parseFixedPoint(t, a[0]) to := from if len(a) > 1 { var toScale int if to, toScale = parseFixedPoint(t, a[1]); toScale != scale { t.Fatalf("%s:%s: non-matching scales %d versus %d", loc, s, scale, toScale) } } m := 1 for i := 0; i < scale; i++ { m *= 10 } for n := from; n <= to; n++ { if f := matchPlural(p, langIndex, n/m, n%m, scale); f != tc.form { t.Errorf("%[1]s:dec(%[2]d.%0[4]*[3]d) = %[5]v; want %[6]v", loc, n/m, n%m, scale, f, tc.form) } } } } } }
// genSymbols generates the symbols used for currencies. Most symbols are // defined in root and there is only very small variation per language. // The following rules apply: // - A symbol can be requested as normal or narrow. // - If a symbol is not defined for a currency, it defaults to its ISO code. func (b *builder) genSymbols(w *gen.CodeWriter, data *cldr.CLDR) { d, err := cldr.ParseDraft(*draft) if err != nil { log.Fatalf("filter: %v", err) } const ( normal = iota narrow numTypes ) // language -> currency -> type -> symbol var symbols [language.NumCompactTags][][numTypes]*string // Collect symbol information per language. for _, lang := range data.Locales() { ldml := data.RawLDML(lang) if ldml.Numbers == nil || ldml.Numbers.Currencies == nil { continue } langIndex, ok := language.CompactIndex(language.MustParse(lang)) if !ok { log.Fatalf("No compact index for language %s", lang) } symbols[langIndex] = make([][numTypes]*string, b.numCurrencies+1) for _, c := range ldml.Numbers.Currencies.Currency { syms := cldr.MakeSlice(&c.Symbol) syms.SelectDraft(d) for _, sym := range c.Symbol { v := sym.Data() if v == c.Type { // We define "" to mean the ISO symbol. v = "" } cur := b.currencies.Index([]byte(c.Type)) // XXX gets reassigned to 0 in the package's code. if c.Type == "XXX" { cur = 0 } if cur == -1 { fmt.Println("Unsupported:", c.Type) continue } switch sym.Alt { case "": symbols[langIndex][cur][normal] = &v case "narrow": symbols[langIndex][cur][narrow] = &v } } } } // Remove values identical to the parent. for langIndex, data := range symbols { for curIndex, curs := range data { for typ, sym := range curs { if sym == nil { continue } for p := uint16(langIndex); p != 0; { p = internal.Parent[p] x := symbols[p] if x == nil { continue } if v := x[curIndex][typ]; v != nil || p == 0 { // Value is equal to the default value root value is undefined. parentSym := "" if v != nil { parentSym = *v } if parentSym == *sym { // Value is the same as parent. data[curIndex][typ] = nil } break } } } } } // Create symbol index. symbolData := []byte{0} symbolLookup := map[string]uint16{"": 0} // 0 means default, so block that value. for _, data := range symbols { for _, curs := range data { for _, sym := range curs { if sym == nil { continue } if _, ok := symbolLookup[*sym]; !ok { symbolLookup[*sym] = uint16(len(symbolData)) symbolData = append(symbolData, byte(len(*sym))) symbolData = append(symbolData, *sym...) } } } } w.WriteComment(` symbols holds symbol data of the form <n> <str>, where n is the length of the symbol string str.`) w.WriteConst("symbols", string(symbolData)) // Create index from language to currency lookup to symbol. type curToIndex struct{ cur, idx uint16 } w.WriteType(curToIndex{}) prefix := []string{"normal", "narrow"} // Create data for regular and narrow symbol data. for typ := normal; typ <= narrow; typ++ { indexes := []curToIndex{} // maps currency to symbol index languages := []uint16{} for _, data := range symbols { languages = append(languages, uint16(len(indexes))) for curIndex, curs := range data { if sym := curs[typ]; sym != nil { indexes = append(indexes, curToIndex{uint16(curIndex), symbolLookup[*sym]}) } } } languages = append(languages, uint16(len(indexes))) w.WriteVar(prefix[typ]+"LangIndex", languages) w.WriteVar(prefix[typ]+"SymIndex", indexes) } }
func genSymbols(w *gen.CodeWriter, data *cldr.CLDR) { d, err := cldr.ParseDraft(*draft) if err != nil { log.Fatalf("invalid draft level: %v", err) } nNumberSystems := system(len(systemMap)) type symbols [NumSymbolTypes]string type key struct { tag int // from language.CompactIndex system system } symbolMap := map[key]*symbols{} defaults := map[int]system{} for _, lang := range data.Locales() { ldml := data.RawLDML(lang) if ldml.Numbers == nil { continue } langIndex, ok := language.CompactIndex(language.MustParse(lang)) if !ok { log.Fatalf("No compact index for language %s", lang) } if d := ldml.Numbers.DefaultNumberingSystem; len(d) > 0 { defaults[langIndex] = getNumberSystem(d[0].Data()) } syms := cldr.MakeSlice(&ldml.Numbers.Symbols) syms.SelectDraft(d) for _, sym := range ldml.Numbers.Symbols { if sym.NumberSystem == "" { // This is just linking the default of root to "latn". continue } symbolMap[key{langIndex, getNumberSystem(sym.NumberSystem)}] = &symbols{ SymDecimal: getFirst("decimal", sym.Decimal), SymGroup: getFirst("group", sym.Group), SymList: getFirst("list", sym.List), SymPercentSign: getFirst("percentSign", sym.PercentSign), SymPlusSign: getFirst("plusSign", sym.PlusSign), SymMinusSign: getFirst("minusSign", sym.MinusSign), SymExponential: getFirst("exponential", sym.Exponential), SymSuperscriptingExponent: getFirst("superscriptingExponent", sym.SuperscriptingExponent), SymPerMille: getFirst("perMille", sym.PerMille), SymInfinity: getFirst("infinity", sym.Infinity), SymNan: getFirst("nan", sym.Nan), SymTimeSeparator: getFirst("timeSeparator", sym.TimeSeparator), } } } // Expand all values. for k, syms := range symbolMap { for t := SymDecimal; t < NumSymbolTypes; t++ { p := k.tag for syms[t] == "" { p = int(internal.Parent[p]) if pSyms, ok := symbolMap[key{p, k.system}]; ok && (*pSyms)[t] != "" { syms[t] = (*pSyms)[t] break } if p == 0 /* und */ { // Default to root, latn. syms[t] = (*symbolMap[key{}])[t] } } } } // Unique the symbol sets and write the string data. m := map[symbols]int{} sb := stringset.NewBuilder() symIndex := [][NumSymbolTypes]byte{} for ns := system(0); ns < nNumberSystems; ns++ { for _, l := range data.Locales() { langIndex, _ := language.CompactIndex(language.MustParse(l)) s := symbolMap[key{langIndex, ns}] if s == nil { continue } if _, ok := m[*s]; !ok { m[*s] = len(symIndex) sb.Add(s[:]...) var x [NumSymbolTypes]byte for i := SymDecimal; i < NumSymbolTypes; i++ { x[i] = byte(sb.Index((*s)[i])) } symIndex = append(symIndex, x) } } } w.WriteVar("symIndex", symIndex) w.WriteVar("symData", sb.Set()) // resolveSymbolIndex gets the index from the closest matching locale, // including the locale itself. resolveSymbolIndex := func(langIndex int, ns system) byte { for { if sym := symbolMap[key{langIndex, ns}]; sym != nil { return byte(m[*sym]) } if langIndex == 0 { return 0 // und, latn } langIndex = int(internal.Parent[langIndex]) } } // Create an index with the symbols for each locale for the latn numbering // system. If this is not the default, or the only one, for a locale, we // will overwrite the value later. var langToDefaults [language.NumCompactTags]byte for _, l := range data.Locales() { langIndex, _ := language.CompactIndex(language.MustParse(l)) langToDefaults[langIndex] = resolveSymbolIndex(langIndex, 0) } // Delete redundant entries. for _, l := range data.Locales() { langIndex, _ := language.CompactIndex(language.MustParse(l)) def := defaults[langIndex] syms := symbolMap[key{langIndex, def}] if syms == nil { continue } for ns := system(0); ns < nNumberSystems; ns++ { if ns == def { continue } if altSyms, ok := symbolMap[key{langIndex, ns}]; ok && *altSyms == *syms { delete(symbolMap, key{langIndex, ns}) } } } // Create a sorted list of alternatives per language. This will only need to // be referenced if a user specified an alternative numbering system. var langToAlt []altSymData for _, l := range data.Locales() { langIndex, _ := language.CompactIndex(language.MustParse(l)) start := len(langToAlt) if start > 0x7F { log.Fatal("Number of alternative assignments > 0x7F") } // Create the entry for the default value. def := defaults[langIndex] langToAlt = append(langToAlt, altSymData{ compactTag: uint16(langIndex), system: def, symIndex: resolveSymbolIndex(langIndex, def), }) for ns := system(0); ns < nNumberSystems; ns++ { if def == ns { continue } if sym := symbolMap[key{langIndex, ns}]; sym != nil { langToAlt = append(langToAlt, altSymData{ compactTag: uint16(langIndex), system: ns, symIndex: resolveSymbolIndex(langIndex, ns), }) } } if def == 0 && len(langToAlt) == start+1 { // No additional data: erase the entry. langToAlt = langToAlt[:start] } else { // Overwrite the entry in langToDefaults. langToDefaults[langIndex] = 0x80 | byte(start) } } w.WriteComment(` langToDefaults maps a compact language index to the default numbering system and default symbol set`) w.WriteVar("langToDefaults", langToDefaults) w.WriteComment(` langToAlt is a list of numbering system and symbol set pairs, sorted and marked by compact language index.`) w.WriteVar("langToAlt", langToAlt) }
func genPlurals(w *gen.CodeWriter, data *cldr.CLDR) { for _, plurals := range data.Supplemental().Plurals { if plurals.Type == "" { continue } // Initialize setMap and inclusionMasks. They are already populated with // a few entries to serve as an example and to assign nice numbers to // common cases. // setMap contains sets of numbers represented by boolean arrays where // a true value for element i means that the number i is included. setMap := map[[numN]bool]int{ // The above init func adds an entry for including all numbers. [numN]bool{1: true}: 1, // fix {1} to a nice value [numN]bool{2: true}: 2, // fix {2} to a nice value [numN]bool{0: true}: 3, // fix {0} to a nice value } // inclusionMasks contains bit masks for every number under numN to // indicate in which set the number is included. Bit 1 << x will be set // if it is included in set x. inclusionMasks := [numN]uint64{ // Note: these entries are not complete: more bits will be set along the way. 0: 1 << 3, 1: 1 << 1, 2: 1 << 2, } // Create set {0..99}. We will assign this set the identifier 0. var all [numN]bool for i := range all { // Mark number i as being included in the set (which has identifier 0). inclusionMasks[i] |= 1 << 0 // Mark number i as included in the set. all[i] = true } // Register the identifier for the set. setMap[all] = 0 rules := []pluralCheck{} index := []byte{0} langMap := map[int]byte{0: 0} // From compact language index to index for _, pRules := range plurals.PluralRules { // Parse the rules. var conds []orCondition for _, rule := range pRules.PluralRule { form := countMap[rule.Count] conds = parsePluralCondition(conds, rule.Data(), form) } // Encode the rules. for _, c := range conds { // If an or condition only has filters, we create an entry for // this filter and the set that contains all values. empty := true for _, b := range c.used { empty = empty && !b } if empty { rules = append(rules, pluralCheck{ cat: byte(opMod<<opShift) | byte(c.form), setID: 0, // all values }) continue } // We have some entries with values. for i, set := range c.set { if !c.used[i] { continue } index, ok := setMap[set] if !ok { index = len(setMap) setMap[set] = index for i := range inclusionMasks { if set[i] { inclusionMasks[i] |= 1 << uint64(index) } } } rules = append(rules, pluralCheck{ cat: byte(i<<opShift | andNext), setID: byte(index), }) } // Now set the last entry to the plural form the rule matches. rules[len(rules)-1].cat &^= formMask rules[len(rules)-1].cat |= byte(c.form) } // Point the relevant locales to the created entries. for _, loc := range strings.Split(pRules.Locales, " ") { if strings.TrimSpace(loc) == "" { continue } lang, ok := language.CompactIndex(language.MustParse(loc)) if !ok { log.Printf("No compact index for locale %q", loc) } langMap[lang] = byte(len(index) - 1) } index = append(index, byte(len(rules))) } w.WriteVar(plurals.Type+"Rules", rules) w.WriteVar(plurals.Type+"Index", index) // Expand the values. langToIndex := make([]byte, language.NumCompactTags) for i := range langToIndex { for p := i; ; p = int(internal.Parent[p]) { if x, ok := langMap[p]; ok { langToIndex[i] = x break } } } w.WriteVar(plurals.Type+"LangToIndex", langToIndex) // Need to convert array to slice because of golang.org/issue/7651. // This will allow tables to be dropped when unused. This is especially // relevant for the ordinal data, which I suspect won't be used as much. w.WriteVar(plurals.Type+"InclusionMasks", inclusionMasks[:]) if len(rules) > 0xFF { log.Fatalf("Too many entries for rules: %#x", len(rules)) } if len(index) > 0xFF { log.Fatalf("Too many entries for index: %#x", len(index)) } if len(setMap) > 64 { // maximum number of bits. log.Fatalf("Too many entries for setMap: %d", len(setMap)) } w.WriteComment( "Slots used for %s: %X of 0xFF rules; %X of 0xFF indexes; %d of 64 sets", plurals.Type, len(rules), len(index), len(setMap)) // Prevent comment from attaching to the next entry. fmt.Fprint(w, "\n\n") } }
// genFormats generates the lookup table for decimal, scientific and percent // patterns. // // CLDR allows for patterns to be different per language for different numbering // systems. In practice the patterns are set to be consistent for a language // independent of the numbering system. genFormats verifies that no language // deviates from this. func genFormats(w *gen.CodeWriter, data *cldr.CLDR) { d, err := cldr.ParseDraft(*draft) if err != nil { log.Fatalf("invalid draft level: %v", err) } // Fill the first slot with a dummy so we can identify unspecified tags. formats := []number.Format{{}} patterns := map[string]int{} // TODO: It would be possible to eliminate two of these slices by having // another indirection and store a reference to the combination of patterns. decimal := make([]byte, language.NumCompactTags) scientific := make([]byte, language.NumCompactTags) percent := make([]byte, language.NumCompactTags) for _, lang := range data.Locales() { ldml := data.RawLDML(lang) if ldml.Numbers == nil { continue } langIndex, ok := language.CompactIndex(language.MustParse(lang)) if !ok { log.Fatalf("No compact index for language %s", lang) } type patternSlice []*struct { cldr.Common Numbers string `xml:"numbers,attr"` Count string `xml:"count,attr"` } add := func(name string, tags []byte, ps patternSlice) { sl := cldr.MakeSlice(&ps) sl.SelectDraft(d) if len(ps) == 0 { return } if len(ps) > 2 || len(ps) == 2 && ps[0] != ps[1] { log.Fatalf("Inconsistent %d patterns for language %s", name, lang) } s := ps[0].Data() index, ok := patterns[s] if !ok { nf, err := number.ParsePattern(s) if err != nil { log.Fatal(err) } index = len(formats) patterns[s] = index formats = append(formats, *nf) } tags[langIndex] = byte(index) } for _, df := range ldml.Numbers.DecimalFormats { for _, l := range df.DecimalFormatLength { if l.Type != "" { continue } for _, f := range l.DecimalFormat { add("decimal", decimal, f.Pattern) } } } for _, df := range ldml.Numbers.ScientificFormats { for _, l := range df.ScientificFormatLength { if l.Type != "" { continue } for _, f := range l.ScientificFormat { add("scientific", scientific, f.Pattern) } } } for _, df := range ldml.Numbers.PercentFormats { for _, l := range df.PercentFormatLength { if l.Type != "" { continue } for _, f := range l.PercentFormat { add("percent", percent, f.Pattern) } } } } // Complete the parent tag array to reflect inheritance. An index of 0 // indicates an unspecified value. for _, data := range [][]byte{decimal, scientific, percent} { for i := range data { p := uint16(i) for ; data[p] == 0; p = internal.Parent[p] { } data[i] = data[p] } } w.WriteVar("tagToDecimal", decimal) w.WriteVar("tagToScientific", scientific) w.WriteVar("tagToPercent", percent) value := strings.Replace(fmt.Sprintf("%#v", formats), "number.", "", -1) // Break up the lines. This won't give ideal perfect formatting, but it is // better than one huge line. value = strings.Replace(value, ", ", ",\n", -1) fmt.Fprintf(w, "var formats = %s\n", value) }
func TestSymbols(t *testing.T) { testtext.SkipIfNotLong(t) draft, err := cldr.ParseDraft(*draft) if err != nil { log.Fatalf("invalid draft level: %v", err) } r := gen.OpenCLDRCoreZip() defer r.Close() d := &cldr.Decoder{} d.SetDirFilter("main") d.SetSectionFilter("numbers") data, err := d.DecodeZip(r) if err != nil { t.Fatalf("DecodeZip: %v", err) } for _, lang := range data.Locales() { ldml := data.RawLDML(lang) if ldml.Numbers == nil { continue } langIndex, ok := language.CompactIndex(language.MustParse(lang)) if !ok { t.Fatalf("No compact index for language %s", lang) } syms := cldr.MakeSlice(&ldml.Numbers.Symbols) syms.SelectDraft(draft) for _, sym := range ldml.Numbers.Symbols { if sym.NumberSystem == "" { continue } testCases := []struct { name string st SymbolType x interface{} }{ {"Decimal", SymDecimal, sym.Decimal}, {"Group", SymGroup, sym.Group}, {"List", SymList, sym.List}, {"PercentSign", SymPercentSign, sym.PercentSign}, {"PlusSign", SymPlusSign, sym.PlusSign}, {"MinusSign", SymMinusSign, sym.MinusSign}, {"Exponential", SymExponential, sym.Exponential}, {"SuperscriptingExponent", SymSuperscriptingExponent, sym.SuperscriptingExponent}, {"PerMille", SymPerMille, sym.PerMille}, {"Infinity", SymInfinity, sym.Infinity}, {"NaN", SymNan, sym.Nan}, {"TimeSeparator", SymTimeSeparator, sym.TimeSeparator}, } info := InfoFromLangID(langIndex, sym.NumberSystem) for _, tc := range testCases { // Extract the wanted value. v := reflect.ValueOf(tc.x) if v.Len() == 0 { return } if v.Len() > 1 { t.Fatalf("Multiple values of %q within single symbol not supported.", tc.name) } want := v.Index(0).MethodByName("Data").Call(nil)[0].String() got := info.Symbol(tc.st) if got != want { t.Errorf("%s:%s:%s: got %q; want %q", lang, sym.NumberSystem, tc.name, got, want) } } } } }