Exemplo n.º 1
0
func ExampleCompose() {
	nl, _ := language.ParseBase("nl")
	us, _ := language.ParseRegion("US")
	de := language.Make("de-1901-u-co-phonebk")
	jp := language.Make("ja-JP")
	fi := language.Make("fi-x-ing")

	u, _ := language.ParseExtension("u-nu-arabic")
	x, _ := language.ParseExtension("x-piglatin")

	// Combine a base language and region.
	fmt.Println(language.Compose(nl, us))
	// Combine a base language and extension.
	fmt.Println(language.Compose(nl, x))
	// Replace the region.
	fmt.Println(language.Compose(jp, us))
	// Combine several tags.
	fmt.Println(language.Compose(us, nl, u))

	// Replace the base language of a tag.
	fmt.Println(language.Compose(de, nl))
	fmt.Println(language.Compose(de, nl, u))
	// Remove the base language.
	fmt.Println(language.Compose(de, language.Base{}))
	// Remove all variants.
	fmt.Println(language.Compose(de, []language.Variant{}))
	// Remove all extensions.
	fmt.Println(language.Compose(de, []language.Extension{}))
	fmt.Println(language.Compose(fi, []language.Extension{}))
	// Remove all variants and extensions.
	fmt.Println(language.Compose(de.Raw()))

	// An error is gobbled or returned if non-nil.
	fmt.Println(language.Compose(language.ParseRegion("ZA")))
	fmt.Println(language.Compose(language.ParseRegion("HH")))

	// Compose uses the same Default canonicalization as Make.
	fmt.Println(language.Compose(language.Raw.Parse("en-Latn-UK")))

	// Call compose on a different CanonType for different results.
	fmt.Println(language.All.Compose(language.Raw.Parse("en-Latn-UK")))

	// Output:
	// nl-US <nil>
	// nl-x-piglatin <nil>
	// ja-US <nil>
	// nl-US-u-nu-arabic <nil>
	// nl-1901-u-co-phonebk <nil>
	// nl-1901-u-nu-arabic <nil>
	// und-1901-u-co-phonebk <nil>
	// de-u-co-phonebk <nil>
	// de-1901 <nil>
	// fi <nil>
	// de <nil>
	// und-ZA <nil>
	// und language: subtag "HH" is well-formed but unknown
	// en-Latn-GB <nil>
	// en-GB <nil>
}
Exemplo n.º 2
0
func ExampleTag_Region() {
	ru := language.Make("ru")
	en := language.Make("en")
	fmt.Println(ru.Region())
	fmt.Println(en.Region())
	// Output:
	// RU Low
	// US Low
}
Exemplo n.º 3
0
func ExampleTag_Base() {
	fmt.Println(language.Make("und").Base())
	fmt.Println(language.Make("und-US").Base())
	fmt.Println(language.Make("und-NL").Base())
	fmt.Println(language.Make("und-419").Base()) // Latin America
	fmt.Println(language.Make("und-ZZ").Base())
	// Output:
	// en Low
	// en High
	// nl High
	// es Low
	// en Low
}
Exemplo n.º 4
0
func ExampleTags() {
	n := display.Tags(language.English)
	fmt.Println(n.Name(language.Make("nl")))
	fmt.Println(n.Name(language.Make("nl-BE")))
	fmt.Println(n.Name(language.Make("nl-CW")))
	fmt.Println(n.Name(language.Make("nl-Arab")))
	fmt.Println(n.Name(language.Make("nl-Cyrl-RU")))

	// Output:
	// Dutch
	// Flemish
	// Dutch (Curaçao)
	// Dutch (Arabic)
	// Dutch (Cyrillic, Russia)
}
Exemplo n.º 5
0
//Load load locales from filesystem
func (p *I18n) Load(dir string) error {
	if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		p.Logger.Debugf("Find locale file %s", path)
		if err != nil {
			return err
		}
		if info.Mode().IsRegular() {
			ss := strings.Split(info.Name(), ".")
			if len(ss) != 3 {
				return fmt.Errorf("Ingnore locale file %s", info.Name())
			}
			code := ss[0]
			lang := language.Make(ss[1])

			fd, err := os.Open(path)
			if err != nil {
				return err
			}
			defer fd.Close()
			sc := bufio.NewScanner(fd)
			for sc.Scan() {
				line := sc.Text()
				idx := strings.Index(line, "=")
				if idx <= 0 || line[0] == '#' {
					continue
				}
				p.set(&lang, strings.TrimSpace(code+"."+line[0:idx]), strings.TrimSpace(line[idx+1:len(line)]))
			}

		}
		return nil
	}); err != nil {
		return err
	}

	for lang := range p.Locales {
		lng := language.Make(lang)
		ks, err := p.Provider.Keys(&lng)
		if err != nil {
			return err
		}
		for _, k := range ks {
			p.Locales[lang][k] = p.Provider.Get(&lng, k)
		}
		p.Logger.Debugf("Find locale %s, %d items.", lang, len(p.Locales[lang]))
	}
	return nil
}
Exemplo n.º 6
0
func ExampleParent() {
	p := func(tag string) {
		fmt.Printf("parent(%v): %v\n", tag, language.Make(tag).Parent())
	}
	p("zh-CN")

	// Australian English inherits from World English.
	p("en-AU")

	// If the tag has a different maximized script from its parent, a tag with
	// this maximized script is inserted. This allows different language tags
	// which have the same base language and script in common to inherit from
	// a common set of settings.
	p("zh-HK")

	// If the maximized script of the parent is not identical, CLDR will skip
	// inheriting from it, as it means there will not be many entries in common
	// and inheriting from it is nonsensical.
	p("zh-Hant")

	// The parent of a tag with variants and extensions is the tag with all
	// variants and extensions removed.
	p("de-1994-u-co-phonebk")

	// Remove default script.
	p("de-Latn-LU")

	// Output:
	// parent(zh-CN): zh
	// parent(en-AU): en-001
	// parent(zh-HK): zh-Hant
	// parent(zh-Hant): und
	// parent(de-1994-u-co-phonebk): de
	// parent(de-Latn-LU): de
}
Exemplo n.º 7
0
// parseMain parses XML files in the main directory of the CLDR core.zip file.
func parseMain() {
	d := &cldr.Decoder{}
	d.SetDirFilter("main")
	d.SetSectionFilter("characters")
	data := decodeCLDR(d)
	for _, loc := range data.Locales() {
		x := data.RawLDML(loc)
		if skipLang(x.Identity.Language.Type) {
			continue
		}
		if x.Characters != nil {
			x, _ = data.LDML(loc)
			loc = language.Make(loc).String()
			for _, ec := range x.Characters.ExemplarCharacters {
				if ec.Draft != "" {
					continue
				}
				if _, ok := localeChars[loc]; !ok {
					mainLocales = append(mainLocales, loc)
					localeChars[loc] = make(charSets)
				}
				localeChars[loc][ec.Type] = parseCharacters(ec.Data())
			}
		}
	}
}
Exemplo n.º 8
0
func ExampleCanonType() {
	p := func(id string) {
		fmt.Printf("Default(%s) -> %s\n", id, language.Make(id))
		fmt.Printf("BCP47(%s) -> %s\n", id, language.BCP47.Make(id))
		fmt.Printf("Macro(%s) -> %s\n", id, language.Macro.Make(id))
		fmt.Printf("All(%s) -> %s\n", id, language.All.Make(id))
	}
	p("en-Latn")
	p("sh")
	p("zh-cmn")
	p("bjd")
	p("iw-Latn-fonipa-u-cu-usd")
	// Output:
	// Default(en-Latn) -> en-Latn
	// BCP47(en-Latn) -> en
	// Macro(en-Latn) -> en-Latn
	// All(en-Latn) -> en
	// Default(sh) -> sr-Latn
	// BCP47(sh) -> sh
	// Macro(sh) -> sh
	// All(sh) -> sr-Latn
	// Default(zh-cmn) -> cmn
	// BCP47(zh-cmn) -> cmn
	// Macro(zh-cmn) -> zh
	// All(zh-cmn) -> zh
	// Default(bjd) -> drl
	// BCP47(bjd) -> drl
	// Macro(bjd) -> bjd
	// All(bjd) -> drl
	// Default(iw-Latn-fonipa-u-cu-usd) -> he-Latn-fonipa-u-cu-usd
	// BCP47(iw-Latn-fonipa-u-cu-usd) -> he-Latn-fonipa-u-cu-usd
	// Macro(iw-Latn-fonipa-u-cu-usd) -> iw-Latn-fonipa-u-cu-usd
	// All(iw-Latn-fonipa-u-cu-usd) -> he-Latn-fonipa-u-cu-usd
}
Exemplo n.º 9
0
func (stop *StopWords) stopWordsCount(lang string, text string) wordStats {
	if text == "" {
		return wordStats{}
	}
	ws := wordStats{}
	stopWords := set.New()
	text = strings.ToLower(text)

	tokenizer := NewMultilangTokenizer(language.Make(lang))
	items := tokenizer.Tokenize(text)
	stops := stop.cachedStopWords[lang]
	if stops != nil {
		for _, item := range items {
			if stops.Has(item) {
				stopWords.Add(item)
			}
		}
	}

	ws.stopWordCount = stopWords.Size()
	ws.wordCount = len(items)
	ws.stopWords = stopWords

	return ws
}
Exemplo n.º 10
0
func TestTokenizer(t *testing.T) {
	japanese := language.Make("en")
	tokenizer := NewMultilangTokenizer(japanese)
	tokens := tokenizer.Tokenize("Language and Locale Matching in Go")
	if !reflect.DeepEqual(tokens, []string{"Language", "and", "Locale", "Matching", "in", "Go"}) {
		t.Fatalf("cannot tokenize english string. tokens %v", tokens)
	}
}
Exemplo n.º 11
0
func TestJapaneseTokenizer(t *testing.T) {
	japanese := language.Make("ja")
	tokenizer := NewMultilangTokenizer(japanese)
	tokens := tokenizer.Tokenize("すもももももももものうち")
	if !reflect.DeepEqual(tokens, []string{"すもも", "も", "もも", "も", "もも", "の", "うち"}) {
		t.Fatalf("cannot tokenize japanese string. tokens %v", tokens)
	}
}
Exemplo n.º 12
0
func ExampleTag_Script() {
	en := language.Make("en")
	sr := language.Make("sr")
	sr_Latn := language.Make("sr_Latn")
	fmt.Println(en.Script())
	fmt.Println(sr.Script())
	// Was a script explicitly specified?
	_, c := sr.Script()
	fmt.Println(c == language.Exact)
	_, c = sr_Latn.Script()
	fmt.Println(c == language.Exact)
	// Output:
	// Latn High
	// Cyrl Low
	// false
	// true
}
Exemplo n.º 13
0
// Supported returns the list of languages for which collating differs from its parent.
func Supported() []language.Tag {
	ids := strings.Split(availableLocales, ",")
	tags := make([]language.Tag, len(ids))
	for i, s := range ids {
		tags[i] = language.Make(s)
	}
	return tags
}
Exemplo n.º 14
0
func checkLang(srcStr, targStr string) tree.Bool {
	srcLang := language.Make(srcStr)
	srcRegion, srcRegionConf := srcLang.Region()

	targLang := language.Make(targStr)
	targRegion, targRegionConf := targLang.Region()

	if srcRegionConf == language.Exact && targRegionConf != language.Exact {
		return tree.Bool(false)
	}

	if srcRegion != targRegion && srcRegionConf == language.Exact && targRegionConf == language.Exact {
		return tree.Bool(false)
	}

	_, _, conf := language.NewMatcher([]language.Tag{srcLang}).Match(targLang)
	return tree.Bool(conf >= language.High)
}
Exemplo n.º 15
0
Arquivo: sort.go Projeto: admpub/i18n
// getCollator returns a collate package Collator pointer. This can result in a
// panic, so this function must recover from that if it happens.
func getCollator(locale string) *collate.Collator {

	defer func() {
		recover()
	}()

	tag := language.Make(locale)

	if tag == language.Und {
		return nil
	}
	return collate.New(tag)
}
Exemplo n.º 16
0
// ExampleDictionary shows how to reduce the amount of data linked into your
// binary by only using the predefined Dictionary variables of the languages you
// wish to support.
func ExampleDictionary() {
	tags := []language.Tag{
		language.English,
		language.German,
		language.Japanese,
		language.Russian,
	}
	dicts := []*display.Dictionary{
		display.English,
		display.German,
		display.Japanese,
		display.Russian,
	}

	m := language.NewMatcher(tags)

	getDict := func(t language.Tag) *display.Dictionary {
		_, i, confidence := m.Match(t)
		// Skip this check if you want to support a fall-back language, which
		// will be the first one passed to NewMatcher.
		if confidence == language.No {
			return nil
		}
		return dicts[i]
	}

	// The matcher will match Swiss German to German.
	n := getDict(language.Make("gsw")).Languages()
	fmt.Println(n.Name(language.German))
	fmt.Println(n.Name(language.Make("de-CH")))
	fmt.Println(n.Name(language.Make("gsw")))

	// Output:
	// Deutsch
	// Schweizer Hochdeutsch
	// Schweizerdeutsch
}
Exemplo n.º 17
0
func ExampleTag_ComprehensibleTo() {
	// Various levels of comprehensibility.
	fmt.Println(language.English.ComprehensibleTo(language.English))
	fmt.Println(language.BritishEnglish.ComprehensibleTo(language.AmericanEnglish))

	// An explicit Und results in no match.
	fmt.Println(language.Und.ComprehensibleTo(language.English))

	fmt.Println("----")

	// There is usually no mutual comprehensibility between different scripts.
	fmt.Println(language.English.ComprehensibleTo(language.Make("en-Dsrt")))

	// One exception is for Traditional versus Simplified Chinese, albeit with
	// a low confidence.
	fmt.Println(language.SimplifiedChinese.ComprehensibleTo(language.TraditionalChinese))

	fmt.Println("----")

	// A Swiss German speaker will often understand High German.
	fmt.Println(language.Make("de").ComprehensibleTo(language.Make("gsw")))

	// The converse is not generally the case.
	fmt.Println(language.Make("gsw").ComprehensibleTo(language.Make("de")))

	// Output:
	// Exact
	// High
	// No
	// ----
	// No
	// Low
	// ----
	// High
	// No
}
Exemplo n.º 18
0
// parseCollation parses XML files in the collation directory of the CLDR core.zip file.
func parseCollation(b *build.Builder) {
	d := &cldr.Decoder{}
	d.SetDirFilter("collation")
	data := decodeCLDR(d)
	for _, loc := range data.Locales() {
		x, err := data.LDML(loc)
		failOnError(err)
		if skipLang(x.Identity.Language.Type) {
			continue
		}
		cs := x.Collations.Collation
		sl := cldr.MakeSlice(&cs)
		if len(types.s) == 0 {
			sl.SelectAnyOf("type", x.Collations.Default())
		} else if !types.all {
			sl.SelectAnyOf("type", types.s...)
		}
		sl.SelectOnePerGroup("alt", altInclude())

		for _, c := range cs {
			id, err := language.Parse(loc)
			if err != nil {
				if loc == "en-US-posix" {
					fmt.Fprintf(os.Stderr, "invalid locale: %q", err.Error())
					continue
				}
				id = language.Make("en-US-u-va-posix")
			}
			// Support both old- and new-style defaults.
			d := c.Type
			if x.Collations.DefaultCollation == nil {
				d = x.Collations.Default()
			} else {
				d = x.Collations.DefaultCollation.Data()
			}
			// We assume tables are being built either for search or collation,
			// but not both. For search the default is always "search".
			if d != c.Type && c.Type != "search" {
				id, err = id.SetTypeForKey("co", c.Type)
				failOnError(err)
			}
			t := b.Tailoring(id)
			c.Process(processor{t})
		}
	}
}
Exemplo n.º 19
0
func ExampleParseAcceptLanguage() {
	// Tags are reordered based on their q rating. A missing q value means 1.0.
	fmt.Println(language.ParseAcceptLanguage(" nn;q=0.3, en-gb;q=0.8, en,"))

	m := language.NewMatcher([]language.Tag{language.Norwegian, language.Make("en-AU")})

	t, _, _ := language.ParseAcceptLanguage("da, en-gb;q=0.8, en;q=0.7")
	fmt.Println(m.Match(t...))

	// Danish is pretty close to Norwegian.
	t, _, _ = language.ParseAcceptLanguage(" da, nl")
	fmt.Println(m.Match(t...))

	// Output:
	// [en en-GB nn] [1 0.8 0.3] <nil>
	// en-AU 1 High
	// no 0 High
}
Exemplo n.º 20
0
Arquivo: i18n.go Projeto: ZiRo-/srndv2
//Read all .ini files in dir, where the filenames are BCP 47 tags
//Use the language matcher to get the best match for the locale preference
func InitI18n(locale, dir string) {
	pref := language.Make(locale) // falls back to en-US on parse error

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	serverLangs := make([]language.Tag, 1)
	serverLangs[0] = language.AmericanEnglish // en-US fallback
	for _, file := range files {
		if filepath.Ext(file.Name()) == ".ini" {
			name := strings.TrimSuffix(file.Name(), ".ini")
			tag, err := language.Parse(name)
			if err == nil {
				serverLangs = append(serverLangs, tag)
			}
		}
	}
	matcher := language.NewMatcher(serverLangs)
	tag, _, _ := matcher.Match(pref)

	fname := filepath.Join(dir, tag.String()+".ini")
	conf, err := configparser.Read(fname)
	if err != nil {
		log.Fatal("cannot read translation file for", tag.String(), err)
	}

	formats, err := conf.Section("formats")
	if err != nil {
		log.Fatal("Cannot read formats sections in translations for", tag.String(), err)
	}
	translations, err := conf.Section("strings")
	if err != nil {
		log.Fatal("Cannot read strings sections in translations for", tag.String(), err)
	}

	i18nProvider = &i18n{
		translation_dir: dir,
		formats:         formats.Options(),
		translations:    translations.Options(),
		locale:          tag,
	}
}
Exemplo n.º 21
0
func (g *phraseGenerator) init(id string) {
	ec := exemplarCharacters
	loc := language.Make(id).String()
	// get sets for locale or parent locale if the set is not defined.
	for i := range g.sets {
		for p, ok := loc, true; ok; p, ok = parent(p) {
			if set, ok := ec[p]; ok && set[i] != "" {
				g.sets[i].set = strings.Split(set[i], " ")
				break
			}
		}
	}
	r := newRewriter()
	r.addCases = *cases
	for i := range g.sets {
		g.sets[i].set = r.rewrite(g.sets[i].set)
	}
	// compute indexes
	for i, set := range g.sets {
		g.n += len(set.set)
		g.sets[i].charIndex = g.n
	}
}
Exemplo n.º 22
0
//Match get language.Tag from string
func Match(lng string) language.Tag {
	tag, _, _ := matcher.Match(language.Make(lng))
	return tag
}
Exemplo n.º 23
0
func newGoCollator(loc string) (Collator, error) {
	c := &goCollator{c: collate.New(language.Make(loc))}
	return c, nil
}
Exemplo n.º 24
0
package main

import (
	"C"
	"golang.org/x/text/language"
	"log"
)

var (
	mather = language.NewMatcher([]language.Tag{language.Make("en"), language.Make("ja"), language.Make("zh-TW"), language.Make("zh-CN")})
)

//export preferredLanguageFrom
func preferredLanguageFrom(httpAcceptLanguage *string) *string {

	tag, _, _ := language.ParseAcceptLanguage(*httpAcceptLanguage)
	t, _, _ := mather.Match(tag...)
	l := t.String()
	return &l
}

//export preferredLanguageFromUseCString
func preferredLanguageFromUseCString(cHttpAcceptLanguage *C.char) *C.char {
	httpAcceptLanguage := C.GoString(cHttpAcceptLanguage)

	tag, _, _ := language.ParseAcceptLanguage(httpAcceptLanguage)
	t, _, _ := mather.Match(tag...)
	return C.CString(t.String())
}

func init() {
Exemplo n.º 25
0
// Clean removes useless spaces and stop words from a byte slice.
// BCP 47 or ISO 639-1 language code (if unknown, we'll apply english filters).
// If cleanHTML is TRUE, remove HTML tags from content and unescape HTML entities.
func Clean(content []byte, langCode string, cleanHTML bool) []byte {
	//Remove HTML tags
	if cleanHTML {
		content = remTags.ReplaceAll(content, []byte(" "))
		content = []byte(html.UnescapeString(string(content)))
	}

	//Parse language
	tag := language.Make(langCode)
	base, _ := tag.Base()
	langCode = base.String()

	//Remove stop words by using a list of most frequent words
	switch langCode {
	case "ar":
		content = removeStopWords(content, arabic)
	case "bg":
		content = removeStopWords(content, bulgarian)
	case "cs":
		content = removeStopWords(content, czech)
	case "da":
		content = removeStopWords(content, danish)
	case "de":
		content = removeStopWords(content, german)
	case "el":
		content = removeStopWords(content, greek)
	case "en":
		content = removeStopWords(content, english)
	case "es":
		content = removeStopWords(content, spanish)
	case "fa":
		content = removeStopWords(content, persian)
	case "fr":
		content = removeStopWords(content, french)
	case "fi":
		content = removeStopWords(content, finnish)
	case "hu":
		content = removeStopWords(content, hungarian)
	case "it":
		content = removeStopWords(content, italian)
	case "ja":
		content = removeStopWords(content, japanese)
	case "lv":
		content = removeStopWords(content, latvian)
	case "nl":
		content = removeStopWords(content, dutch)
	case "no":
		content = removeStopWords(content, norwegian)
	case "pl":
		content = removeStopWords(content, polish)
	case "pt":
		content = removeStopWords(content, portuguese)
	case "ro":
		content = removeStopWords(content, romanian)
	case "ru":
		content = removeStopWords(content, russian)
	case "sk":
		content = removeStopWords(content, slovak)
	case "sv":
		content = removeStopWords(content, swedish)
	case "th":
		content = removeStopWords(content, thai)
	case "tr":
		content = removeStopWords(content, turkish)
	}

	//Remove duplicated space characters
	content = oneSpace.ReplaceAll(content, []byte(" "))

	return content
}
Exemplo n.º 26
0
func main() {
	gen.Init()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer func() {
		buf := &bytes.Buffer{}

		if _, err = w.WriteGo(buf, "language"); err != nil {
			log.Fatalf("Error formatting file index.go: %v", err)
		}

		// Since we're generating a table for our own package we need to rewrite
		// doing the equivalent of go fmt -r 'language.b -> b'. Using
		// bytes.Replace will do.
		out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
		if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
			log.Fatalf("Could not create file index.go: %v", err)
		}
	}()

	m := map[language.Tag]bool{}
	for _, lang := range data.Locales() {
		if x := data.RawLDML(lang); false ||
			x.LocaleDisplayNames != nil ||
			x.Characters != nil ||
			x.Delimiters != nil ||
			x.Measurement != nil ||
			x.Dates != nil ||
			x.Numbers != nil ||
			x.Units != nil ||
			x.ListPatterns != nil ||
			x.Collations != nil ||
			x.Segmentations != nil ||
			x.Rbnf != nil ||
			x.Annotations != nil ||
			x.Metadata != nil {

			// TODO: support POSIX natively, albeit non-standard.
			tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
			m[tag] = true
		}
	}
	var core, special []language.Tag

	for t := range m {
		if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
			log.Fatalf("Unexpected extension %v in %v", x, t)
		}
		if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
			core = append(core, t)
		} else {
			special = append(special, t)
		}
	}

	w.WriteComment(`
	NumCompactTags is the number of common tags. The maximum tag is
	NumCompactTags-1.`)
	w.WriteConst("NumCompactTags", len(core)+len(special))

	sort.Sort(byAlpha(special))
	w.WriteVar("specialTags", special)

	type coreKey struct {
		base   language.Base
		script language.Script
		region language.Region
	}
	w.WriteType(coreKey{})

	// TODO: order by frequency?
	sort.Sort(byAlpha(core))

	// Size computations are just an estimate.
	w.Size += int(reflect.TypeOf(map[coreKey]uint16{}).Size())
	w.Size += len(core) * int(reflect.TypeOf(coreKey{}).Size()+2) // 2 is for uint16

	fmt.Fprintln(w, "var coreTags = map[coreKey]uint16{")
	fmt.Fprintln(w, "coreKey{}: 0, // und")
	i := len(special) + 1 // Und and special tags already written.
	for _, t := range core {
		if t == language.Und {
			continue
		}
		fmt.Fprint(w.Hash, t, i)
		b, s, r := t.Raw()
		key := fmt.Sprintf("%#v", coreKey{b, s, r})
		key = strings.Replace(key[len("main."):], "language.", "", -1)
		fmt.Fprintf(w, "%s: %d, // %s\n", key, i, t)
		i++
	}
	fmt.Fprintln(w, "}")
}
Exemplo n.º 27
0
func main() {
	gen.Init()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer func() {
		buf := &bytes.Buffer{}

		if _, err = w.WriteGo(buf, "language"); err != nil {
			log.Fatalf("Error formatting file index.go: %v", err)
		}

		// Since we're generating a table for our own package we need to rewrite
		// doing the equivalent of go fmt -r 'language.b -> b'. Using
		// bytes.Replace will do.
		out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
		if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
			log.Fatalf("Could not create file index.go: %v", err)
		}
	}()

	m := map[language.Tag]bool{}
	for _, lang := range data.Locales() {
		// We include all locales unconditionally to be consistent with en_US.
		// We want en_US, even though it has no data associated with it.

		// TODO: put any of the languages for which no data exists at the end
		// of the index. This allows all components based on ICU to use that
		// as the cutoff point.
		// if x := data.RawLDML(lang); false ||
		// 	x.LocaleDisplayNames != nil ||
		// 	x.Characters != nil ||
		// 	x.Delimiters != nil ||
		// 	x.Measurement != nil ||
		// 	x.Dates != nil ||
		// 	x.Numbers != nil ||
		// 	x.Units != nil ||
		// 	x.ListPatterns != nil ||
		// 	x.Collations != nil ||
		// 	x.Segmentations != nil ||
		// 	x.Rbnf != nil ||
		// 	x.Annotations != nil ||
		// 	x.Metadata != nil {

		// TODO: support POSIX natively, albeit non-standard.
		tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
		m[tag] = true
		// }
	}
	// Include locales for plural rules, which uses a different structure.
	for _, plurals := range data.Supplemental().Plurals {
		for _, rules := range plurals.PluralRules {
			for _, lang := range strings.Split(rules.Locales, " ") {
				m[language.Make(lang)] = true
			}
		}
	}

	var core, special []language.Tag

	for t := range m {
		if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
			log.Fatalf("Unexpected extension %v in %v", x, t)
		}
		if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
			core = append(core, t)
		} else {
			special = append(special, t)
		}
	}

	w.WriteComment(`
	NumCompactTags is the number of common tags. The maximum tag is
	NumCompactTags-1.`)
	w.WriteConst("NumCompactTags", len(core)+len(special))

	sort.Sort(byAlpha(special))
	w.WriteVar("specialTags", special)

	// TODO: order by frequency?
	sort.Sort(byAlpha(core))

	// Size computations are just an estimate.
	w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size())
	w.Size += len(core) * 6 // size of uint32 and uint16

	fmt.Fprintln(w)
	fmt.Fprintln(w, "var coreTags = map[uint32]uint16{")
	fmt.Fprintln(w, "0x0: 0, // und")
	i := len(special) + 1 // Und and special tags already written.
	for _, t := range core {
		if t == language.Und {
			continue
		}
		fmt.Fprint(w.Hash, t, i)
		b, s, r := t.Raw()
		fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n",
			getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number
			getIndex(s, 2),
			getIndex(r, 3),
			i, t)
		i++
	}
	fmt.Fprintln(w, "}")
}
Exemplo n.º 28
0
// ExampleMatcher_bestMatch gives some examples of getting the best match of
// a set of tags to any of the tags of given set.
func ExampleMatcher() {
	// This is the set of tags from which we want to pick the best match. These
	// can be, for example, the supported languages for some package.
	tags := []language.Tag{
		language.English,
		language.BritishEnglish,
		language.French,
		language.Afrikaans,
		language.BrazilianPortuguese,
		language.EuropeanPortuguese,
		language.Croatian,
		language.SimplifiedChinese,
		language.Raw.Make("iw-IL"),
		language.Raw.Make("iw"),
		language.Raw.Make("he"),
	}
	m := language.NewMatcher(tags)

	// A simple match.
	fmt.Println(m.Match(language.Make("fr")))

	// Australian English is closer to British than American English.
	fmt.Println(m.Match(language.Make("en-AU")))

	// Default to the first tag passed to the Matcher if there is no match.
	fmt.Println(m.Match(language.Make("ar")))

	// Get the default tag.
	fmt.Println(m.Match())

	fmt.Println("----")

	// Croatian speakers will likely understand Serbian written in Latin script.
	fmt.Println(m.Match(language.Make("sr-Latn")))

	// We match SimplifiedChinese, but with Low confidence.
	fmt.Println(m.Match(language.TraditionalChinese))

	// Serbian in Latin script is a closer match to Croatian than Traditional
	// Chinese to Simplified Chinese.
	fmt.Println(m.Match(language.TraditionalChinese, language.Make("sr-Latn")))

	fmt.Println("----")

	// In case a multiple variants of a language are available, the most spoken
	// variant is typically returned.
	fmt.Println(m.Match(language.Portuguese))

	// Pick the first value passed to Match in case of a tie.
	fmt.Println(m.Match(language.Dutch, language.Make("fr-BE"), language.Make("af-NA")))
	fmt.Println(m.Match(language.Dutch, language.Make("af-NA"), language.Make("fr-BE")))

	fmt.Println("----")

	// If a Matcher is initialized with a language and it's deprecated version,
	// it will distinguish between them.
	fmt.Println(m.Match(language.Raw.Make("iw")))

	// However, for non-exact matches, it will treat deprecated versions as
	// equivalent and consider other factors first.
	fmt.Println(m.Match(language.Raw.Make("he-IL")))

	// Output:
	// fr 2 Exact
	// en-GB 1 High
	// en 0 No
	// en 0 No
	// ----
	// hr 6 High
	// zh-Hans 7 Low
	// hr 6 High
	// ----
	// pt-BR 4 High
	// fr 2 High
	// af 3 High
	// ----
	// iw 9 Exact
	// iw-IL 8 Exact
}
Exemplo n.º 29
0
//Ts translate by lang
func (p *I18n) Ts(lng string, code string, args ...interface{}) string {
	l := language.Make(lng)
	return p.T(&l, code, args...)
}
Exemplo n.º 30
0
// +build OMIT

package main

import (
	"fmt"

	"golang.org/x/text/language"
	"golang.org/x/text/language/display"
)

var userPrefs = []language.Tag{
	language.Make("gsw"), // Swiss German
	language.Make("fr"),  // French
}

var serverLangs = []language.Tag{
	language.AmericanEnglish, // en-US fallback
	language.German,          // de
}

var matcher = language.NewMatcher(serverLangs)

func main() {
	tag, index, confidence := matcher.Match(userPrefs...)

	fmt.Printf("best match: %s (%s) index=%d confidence=%v\n",
		display.English.Tags().Name(tag),
		display.Self.Name(tag),
		index, confidence)
	// best match: German (Deutsch) index=1 confidence=High