Пример #1
0
func genPluralsTests(w *gen.CodeWriter, data *cldr.CLDR) {
	w.WriteType(pluralTest{})

	for _, plurals := range data.Supplemental().Plurals {
		if plurals.Type == "" {
			// The empty type is reserved for plural ranges.
			continue
		}
		tests := []pluralTest{}

		for _, pRules := range plurals.PluralRules {
			for _, rule := range pRules.PluralRule {
				test := pluralTest{
					locales: pRules.Locales,
					form:    countMap[rule.Count],
				}
				scan := bufio.NewScanner(strings.NewReader(rule.Data()))
				scan.Split(splitTokens)
				var p *[]string
				for scan.Scan() {
					switch t := scan.Text(); t {
					case "@integer":
						p = &test.integer
					case "@decimal":
						p = &test.decimal
					case ",", "…":
					default:
						if p != nil {
							*p = append(*p, t)
						}
					}
				}
				tests = append(tests, test)
			}
		}
		w.WriteVar(plurals.Type+"Tests", tests)
	}
}
Пример #2
0
Файл: gen.go Проект: jak-atx/vic
func genNumSystem(w *gen.CodeWriter, data *cldr.CLDR) {
	numSysData := []systemData{
		{digitSize: 1, zero: [4]byte{'0'}},
	}

	for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
		if len(ns.Digits) == 0 {
			continue
		}
		switch ns.Id {
		case "latn":
			// hard-wired
			continue
		case "hanidec":
			// non-consecutive digits: treat as "algorithmic"
			continue
		}

		zero, sz := utf8.DecodeRuneInString(ns.Digits)
		if ns.Digits[sz-1]+9 > 0xBF { // 1011 1111: highest continuation byte
			log.Fatalf("Last byte of zero value overflows for %s", ns.Id)
		}

		i := rune(0)
		for _, r := range ns.Digits {
			// Verify that we can do simple math on the UTF-8 byte sequence
			// of zero to get the digit.
			if zero+i != r {
				// Runes not consecutive.
				log.Fatalf("Digit %d of %s (%U) is not offset correctly from zero value", i, ns.Id, r)
			}
			i++
		}
		var x [utf8.UTFMax]byte
		utf8.EncodeRune(x[:], zero)
		id := system(len(numSysData))
		systemMap[ns.Id] = id
		numSysData = append(numSysData, systemData{
			id:        id,
			digitSize: byte(sz),
			zero:      x,
		})
	}
	w.WriteVar("numSysData", numSysData)

	algoID := system(len(numSysData))
	fmt.Fprintln(w, "const (")
	for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
		id, ok := systemMap[ns.Id]
		if !ok {
			id = algoID
			systemMap[ns.Id] = id
			algoID++
		}
		fmt.Fprintf(w, "num%s = %#x\n", strings.Title(ns.Id), id)
	}
	fmt.Fprintln(w, "numNumberSystems")
	fmt.Fprintln(w, ")")

	fmt.Fprintln(w, "var systemMap = map[string]system{")
	for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
		fmt.Fprintf(w, "%q: num%s,\n", ns.Id, strings.Title(ns.Id))
		w.Size += len(ns.Id) + 16 + 1 // very coarse approximation
	}
	fmt.Fprintln(w, "}")
}
Пример #3
0
func genPlurals(w *gen.CodeWriter, data *cldr.CLDR) {
	for _, plurals := range data.Supplemental().Plurals {
		if plurals.Type == "" {
			continue
		}
		// Initialize setMap and inclusionMasks. They are already populated with
		// a few entries to serve as an example and to assign nice numbers to
		// common cases.

		// setMap contains sets of numbers represented by boolean arrays where
		// a true value for element i means that the number i is included.
		setMap := map[[numN]bool]int{
			// The above init func adds an entry for including all numbers.
			[numN]bool{1: true}: 1, // fix {1} to a nice value
			[numN]bool{2: true}: 2, // fix {2} to a nice value
			[numN]bool{0: true}: 3, // fix {0} to a nice value
		}

		// inclusionMasks contains bit masks for every number under numN to
		// indicate in which set the number is included. Bit 1 << x will be set
		// if it is included in set x.
		inclusionMasks := [numN]uint64{
			// Note: these entries are not complete: more bits will be set along the way.
			0: 1 << 3,
			1: 1 << 1,
			2: 1 << 2,
		}

		// Create set {0..99}. We will assign this set the identifier 0.
		var all [numN]bool
		for i := range all {
			// Mark number i as being included in the set (which has identifier 0).
			inclusionMasks[i] |= 1 << 0
			// Mark number i as included in the set.
			all[i] = true
		}
		// Register the identifier for the set.
		setMap[all] = 0

		rules := []pluralCheck{}
		index := []byte{0}
		langMap := map[int]byte{0: 0} // From compact language index to index

		for _, pRules := range plurals.PluralRules {
			// Parse the rules.
			var conds []orCondition
			for _, rule := range pRules.PluralRule {
				form := countMap[rule.Count]
				conds = parsePluralCondition(conds, rule.Data(), form)
			}
			// Encode the rules.
			for _, c := range conds {
				// If an or condition only has filters, we create an entry for
				// this filter and the set that contains all values.
				empty := true
				for _, b := range c.used {
					empty = empty && !b
				}
				if empty {
					rules = append(rules, pluralCheck{
						cat:   byte(opMod<<opShift) | byte(c.form),
						setID: 0, // all values
					})
					continue
				}
				// We have some entries with values.
				for i, set := range c.set {
					if !c.used[i] {
						continue
					}
					index, ok := setMap[set]
					if !ok {
						index = len(setMap)
						setMap[set] = index
						for i := range inclusionMasks {
							if set[i] {
								inclusionMasks[i] |= 1 << uint64(index)
							}
						}
					}
					rules = append(rules, pluralCheck{
						cat:   byte(i<<opShift | andNext),
						setID: byte(index),
					})
				}
				// Now set the last entry to the plural form the rule matches.
				rules[len(rules)-1].cat &^= formMask
				rules[len(rules)-1].cat |= byte(c.form)
			}
			// Point the relevant locales to the created entries.
			for _, loc := range strings.Split(pRules.Locales, " ") {
				if strings.TrimSpace(loc) == "" {
					continue
				}
				lang, ok := language.CompactIndex(language.MustParse(loc))
				if !ok {
					log.Printf("No compact index for locale %q", loc)
				}
				langMap[lang] = byte(len(index) - 1)
			}
			index = append(index, byte(len(rules)))
		}
		w.WriteVar(plurals.Type+"Rules", rules)
		w.WriteVar(plurals.Type+"Index", index)
		// Expand the values.
		langToIndex := make([]byte, language.NumCompactTags)
		for i := range langToIndex {
			for p := i; ; p = int(internal.Parent[p]) {
				if x, ok := langMap[p]; ok {
					langToIndex[i] = x
					break
				}
			}
		}
		w.WriteVar(plurals.Type+"LangToIndex", langToIndex)
		// Need to convert array to slice because of golang.org/issue/7651.
		// This will allow tables to be dropped when unused. This is especially
		// relevant for the ordinal data, which I suspect won't be used as much.
		w.WriteVar(plurals.Type+"InclusionMasks", inclusionMasks[:])

		if len(rules) > 0xFF {
			log.Fatalf("Too many entries for rules: %#x", len(rules))
		}
		if len(index) > 0xFF {
			log.Fatalf("Too many entries for index: %#x", len(index))
		}
		if len(setMap) > 64 { // maximum number of bits.
			log.Fatalf("Too many entries for setMap: %d", len(setMap))
		}
		w.WriteComment(
			"Slots used for %s: %X of 0xFF rules; %X of 0xFF indexes; %d of 64 sets",
			plurals.Type, len(rules), len(index), len(setMap))
		// Prevent comment from attaching to the next entry.
		fmt.Fprint(w, "\n\n")
	}
}