func genPluralsTests(w *gen.CodeWriter, data *cldr.CLDR) {
w.WriteType(pluralTest{})
for _, plurals := range data.Supplemental().Plurals {
if plurals.Type == "" {
// The empty type is reserved for plural ranges.
continue
}
tests := []pluralTest{}
for _, pRules := range plurals.PluralRules {
for _, rule := range pRules.PluralRule {
test := pluralTest{
locales: pRules.Locales,
form: countMap[rule.Count],
}
scan := bufio.NewScanner(strings.NewReader(rule.Data()))
scan.Split(splitTokens)
var p *[]string
for scan.Scan() {
switch t := scan.Text(); t {
case "@integer":
p = &test.integer
case "@decimal":
p = &test.decimal
case ",", "…":
default:
if p != nil {
*p = append(*p, t)
}
}
}
tests = append(tests, test)
}
}
w.WriteVar(plurals.Type+"Tests", tests)
}
}
func genNumSystem(w *gen.CodeWriter, data *cldr.CLDR) {
numSysData := []systemData{
{digitSize: 1, zero: [4]byte{'0'}},
}
for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
if len(ns.Digits) == 0 {
continue
}
switch ns.Id {
case "latn":
// hard-wired
continue
case "hanidec":
// non-consecutive digits: treat as "algorithmic"
continue
}
zero, sz := utf8.DecodeRuneInString(ns.Digits)
if ns.Digits[sz-1]+9 > 0xBF { // 1011 1111: highest continuation byte
log.Fatalf("Last byte of zero value overflows for %s", ns.Id)
}
i := rune(0)
for _, r := range ns.Digits {
// Verify that we can do simple math on the UTF-8 byte sequence
// of zero to get the digit.
if zero+i != r {
// Runes not consecutive.
log.Fatalf("Digit %d of %s (%U) is not offset correctly from zero value", i, ns.Id, r)
}
i++
}
var x [utf8.UTFMax]byte
utf8.EncodeRune(x[:], zero)
id := system(len(numSysData))
systemMap[ns.Id] = id
numSysData = append(numSysData, systemData{
id: id,
digitSize: byte(sz),
zero: x,
})
}
w.WriteVar("numSysData", numSysData)
algoID := system(len(numSysData))
fmt.Fprintln(w, "const (")
for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
id, ok := systemMap[ns.Id]
if !ok {
id = algoID
systemMap[ns.Id] = id
algoID++
}
fmt.Fprintf(w, "num%s = %#x\n", strings.Title(ns.Id), id)
}
fmt.Fprintln(w, "numNumberSystems")
fmt.Fprintln(w, ")")
fmt.Fprintln(w, "var systemMap = map[string]system{")
for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
fmt.Fprintf(w, "%q: num%s,\n", ns.Id, strings.Title(ns.Id))
w.Size += len(ns.Id) + 16 + 1 // very coarse approximation
}
fmt.Fprintln(w, "}")
}
func genPlurals(w *gen.CodeWriter, data *cldr.CLDR) {
for _, plurals := range data.Supplemental().Plurals {
if plurals.Type == "" {
continue
}
// Initialize setMap and inclusionMasks. They are already populated with
// a few entries to serve as an example and to assign nice numbers to
// common cases.
// setMap contains sets of numbers represented by boolean arrays where
// a true value for element i means that the number i is included.
setMap := map[[numN]bool]int{
// The above init func adds an entry for including all numbers.
[numN]bool{1: true}: 1, // fix {1} to a nice value
[numN]bool{2: true}: 2, // fix {2} to a nice value
[numN]bool{0: true}: 3, // fix {0} to a nice value
}
// inclusionMasks contains bit masks for every number under numN to
// indicate in which set the number is included. Bit 1 << x will be set
// if it is included in set x.
inclusionMasks := [numN]uint64{
// Note: these entries are not complete: more bits will be set along the way.
0: 1 << 3,
1: 1 << 1,
2: 1 << 2,
}
// Create set {0..99}. We will assign this set the identifier 0.
var all [numN]bool
for i := range all {
// Mark number i as being included in the set (which has identifier 0).
inclusionMasks[i] |= 1 << 0
// Mark number i as included in the set.
all[i] = true
}
// Register the identifier for the set.
setMap[all] = 0
rules := []pluralCheck{}
index := []byte{0}
langMap := map[int]byte{0: 0} // From compact language index to index
for _, pRules := range plurals.PluralRules {
// Parse the rules.
var conds []orCondition
for _, rule := range pRules.PluralRule {
form := countMap[rule.Count]
conds = parsePluralCondition(conds, rule.Data(), form)
}
// Encode the rules.
for _, c := range conds {
// If an or condition only has filters, we create an entry for
// this filter and the set that contains all values.
empty := true
for _, b := range c.used {
empty = empty && !b
}
if empty {
rules = append(rules, pluralCheck{
cat: byte(opMod<<opShift) | byte(c.form),
setID: 0, // all values
})
continue
}
// We have some entries with values.
for i, set := range c.set {
if !c.used[i] {
continue
}
index, ok := setMap[set]
if !ok {
index = len(setMap)
setMap[set] = index
for i := range inclusionMasks {
if set[i] {
inclusionMasks[i] |= 1 << uint64(index)
}
}
}
rules = append(rules, pluralCheck{
cat: byte(i<<opShift | andNext),
setID: byte(index),
})
}
// Now set the last entry to the plural form the rule matches.
rules[len(rules)-1].cat &^= formMask
rules[len(rules)-1].cat |= byte(c.form)
}
// Point the relevant locales to the created entries.
for _, loc := range strings.Split(pRules.Locales, " ") {
if strings.TrimSpace(loc) == "" {
continue
}
lang, ok := language.CompactIndex(language.MustParse(loc))
if !ok {
log.Printf("No compact index for locale %q", loc)
}
langMap[lang] = byte(len(index) - 1)
}
index = append(index, byte(len(rules)))
}
w.WriteVar(plurals.Type+"Rules", rules)
w.WriteVar(plurals.Type+"Index", index)
// Expand the values.
langToIndex := make([]byte, language.NumCompactTags)
for i := range langToIndex {
for p := i; ; p = int(internal.Parent[p]) {
if x, ok := langMap[p]; ok {
langToIndex[i] = x
break
}
}
}
w.WriteVar(plurals.Type+"LangToIndex", langToIndex)
// Need to convert array to slice because of golang.org/issue/7651.
// This will allow tables to be dropped when unused. This is especially
// relevant for the ordinal data, which I suspect won't be used as much.
w.WriteVar(plurals.Type+"InclusionMasks", inclusionMasks[:])
if len(rules) > 0xFF {
log.Fatalf("Too many entries for rules: %#x", len(rules))
}
if len(index) > 0xFF {
log.Fatalf("Too many entries for index: %#x", len(index))
}
if len(setMap) > 64 { // maximum number of bits.
log.Fatalf("Too many entries for setMap: %d", len(setMap))
}
w.WriteComment(
"Slots used for %s: %X of 0xFF rules; %X of 0xFF indexes; %d of 64 sets",
plurals.Type, len(rules), len(index), len(setMap))
// Prevent comment from attaching to the next entry.
fmt.Fprint(w, "\n\n")
}
}
