Esempio n. 1
0
// parseCollation parses XML files in the collation directory of the CLDR core.zip file.
func parseCollation(b *build.Builder) {
	d := &cldr.Decoder{}
	d.SetDirFilter("collation")
	data := decodeCLDR(d)
	for _, loc := range data.Locales() {
		x, err := data.LDML(loc)
		failOnError(err)
		if skipLang(x.Identity.Language.Type) {
			continue
		}
		cs := x.Collations.Collation
		sl := cldr.MakeSlice(&cs)
		if !types.all {
			sl.SelectAnyOf("type", append(types.s, x.Collations.Default())...)
		}
		sl.SelectOnePerGroup("alt", altInclude())

		for _, c := range cs {
			m := make(map[locale.Part]string)
			m[locale.TagPart] = loc
			if c.Type != x.Collations.Default() {
				m[locale.Extension('u')] = "co-" + c.Type
			}
			id, err := locale.Compose(m)
			failOnError(err)
			t := b.Tailoring(id)
			c.Process(processor{t})
		}
	}
}
Esempio n. 2
0
func ExampleSlice() {
	var dr *cldr.CLDR // assume this is initalized

	x, _ := dr.LDML("en")
	cs := x.Collations.Collation
	// remove all but the default
	cldr.MakeSlice(&cs).Filter(func(e cldr.Elem) bool {
		return e.GetCommon().Type != x.Collations.Default()
	})
	for i, c := range cs {
		fmt.Println(i, c.Type)
	}
}
Esempio n. 3
0
func (b *builder) filter() {
	filter := func(s *cldr.Slice) {
		if *short {
			s.SelectOnePerGroup("alt", []string{"short", ""})
		} else {
			s.SelectOnePerGroup("alt", []string{"stand-alone", ""})
		}
		d, err := cldr.ParseDraft(*draft)
		if err != nil {
			log.Fatalf("filter: %v", err)
		}
		s.SelectDraft(d)
	}
	for _, loc := range b.data.Locales() {
		if ldn := b.data.RawLDML(loc).LocaleDisplayNames; ldn != nil {
			if ldn.Languages != nil {
				s := cldr.MakeSlice(&ldn.Languages.Language)
				if filter(&s); len(ldn.Languages.Language) == 0 {
					ldn.Languages = nil
				}
			}
			if ldn.Scripts != nil {
				s := cldr.MakeSlice(&ldn.Scripts.Script)
				if filter(&s); len(ldn.Scripts.Script) == 0 {
					ldn.Scripts = nil
				}
			}
			if ldn.Territories != nil {
				s := cldr.MakeSlice(&ldn.Territories.Territory)
				if filter(&s); len(ldn.Territories.Territory) == 0 {
					ldn.Territories = nil
				}
			}
		}
	}
}
Esempio n. 4
0
// writeMatchData writes tables with languages and scripts for which there is
// mutual intelligibility. The data is based on CLDR's languageMatching data.
// Note that we use a different algorithm than the one defined by CLDR and that
// we slightly modify the data. For example, we convert scores to confidence levels.
// We also drop all region-related data as we use a different algorithm to
// determine region equivalence.
func (b *builder) writeMatchData() {
	b.writeType(mutualIntelligibility{})
	b.writeType(scriptIntelligibility{})
	lm := b.supp.LanguageMatching.LanguageMatches
	cldr.MakeSlice(&lm).SelectAnyOf("type", "written")

	matchLang := []mutualIntelligibility{}
	matchScript := []scriptIntelligibility{}
	// Convert the languageMatch entries in lists keyed by desired language.
	for _, m := range lm[0].LanguageMatch {
		// Different versions of CLDR use different separators.
		desired := strings.Replace(m.Desired, "-", "_", -1)
		supported := strings.Replace(m.Supported, "-", "_", -1)
		d := strings.Split(desired, "_")
		s := strings.Split(supported, "_")
		if len(d) != len(s) || len(d) > 2 {
			// Skip all entries with regions and work around CLDR bug.
			continue
		}
		pct, _ := strconv.ParseInt(m.Percent, 10, 8)
		if len(d) == 2 && d[0] == s[0] && len(d[1]) == 4 {
			// language-script pair.
			lang := uint16(0)
			if d[0] != "*" {
				lang = uint16(b.langIndex(d[0]))
			}
			matchScript = append(matchScript, scriptIntelligibility{
				lang: lang,
				want: uint8(b.script.index(d[1])),
				have: uint8(b.script.index(s[1])),
				conf: toConf(uint8(pct)),
			})
			if m.Oneway != "true" {
				matchScript = append(matchScript, scriptIntelligibility{
					lang: lang,
					want: uint8(b.script.index(s[1])),
					have: uint8(b.script.index(d[1])),
					conf: toConf(uint8(pct)),
				})
			}
		} else if len(d) == 1 && d[0] != "*" {
			if pct == 100 {
				// nb == no is already handled by macro mapping. Check there
				// really is only this case.
				if d[0] != "no" || s[0] != "nb" {
					log.Fatalf("unhandled equivalence %s == %s", s[0], d[0])
				}
				continue
			}
			matchLang = append(matchLang, mutualIntelligibility{
				want:   uint16(b.langIndex(d[0])),
				have:   uint16(b.langIndex(s[0])),
				conf:   uint8(pct),
				oneway: m.Oneway == "true",
			})
		} else {
			// TODO: Handle the es_MX -> es_419 mapping. This does not seem to
			// make much sense for our purposes, though.
			a := []string{"*;*", "*_*;*_*", "es_MX;es_419"}
			s := strings.Join([]string{desired, supported}, ";")
			if i := sort.SearchStrings(a, s); i == len(a) || a[i] != s {
				log.Fatalf("%q not handled", s)
			}
		}
	}
	sort.Sort(sortByConf(matchLang))
	// collapse percentage into confidence classes
	for i, m := range matchLang {
		matchLang[i].conf = toConf(m.conf)
	}
	b.writeSlice("matchLang", matchLang)
	b.writeSlice("matchScript", matchScript)
}