コード例 #1
0
ファイル: gen.go プロジェクト: ChongFeng/beats
func main() {
	gen.Init()
	genTables()
	genTests()
	repackage("gen_trieval.go", "trieval.go")
	repackage("gen_common.go", "common_test.go")
}
コード例 #2
0
ファイル: gen.go プロジェクト: glerchundi/kube2nginx
func main() {
	gen.Init()

	const pkg = "number"

	gen.Repackage("gen_common.go", "common.go", pkg)
	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental", "main")
	d.SetSectionFilter("numbers", "numberingSystem")
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer w.WriteGoFile(*outputFile, pkg)

	fmt.Fprintln(w, `import "golang.org/x/text/internal/stringset"`)

	gen.WriteCLDRVersion(w)

	genNumSystem(w, data)
	genSymbols(w, data)
}
コード例 #3
0
ファイル: maketables.go プロジェクト: npchp110/text
func main() {
	gen.Init()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("main", "supplemental")
	d.SetSectionFilter("localeDisplayNames")
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := &bytes.Buffer{}
	gen.WriteCLDRVersion(w)

	b := builder{
		w:     w,
		data:  data,
		group: make(map[string]*group),
	}
	b.generate()
	gen.WriteGoFile(*outputFile, "display", w.Bytes())
}
コード例 #4
0
ファイル: maketables.go プロジェクト: npchp110/text
func main() {
	gen.Init()
	b := build.NewBuilder()
	parseUCA(b)
	if tables.contains("chars") {
		parseMain()
	}
	parseCollation(b)

	c, err := b.Build()
	failOnError(err)

	if *test {
		testCollator(collate.NewFromTable(c))
	} else {
		w := &bytes.Buffer{}

		gen.WriteUnicodeVersion(w)
		gen.WriteCLDRVersion(w)

		if tables.contains("collate") {
			_, err = b.Print(w)
			failOnError(err)
		}
		if tables.contains("chars") {
			printExemplarCharacters(w)
		}
		gen.WriteGoFile("tables.go", *pkg, w.Bytes())
	}
}
コード例 #5
0
func main() {
	gen.Init()

	rewriteCommon()

	w := &bytes.Buffer{}

	b := newBuilder(w)
	fmt.Fprintf(w, version, cldr.Version)

	b.parseIndices()
	b.writeType(fromTo{})
	b.writeLanguage()
	b.writeScript()
	b.writeRegion()
	b.writeVariant()
	// TODO: b.writeLocale()
	b.writeCurrencies()
	b.computeRegionGroups()
	b.writeLikelyData()
	b.writeMatchData()
	b.writeRegionInclusionData()
	b.writeParents()

	fmt.Fprintf(w, "\n// Size: %.1fK (%d bytes); Check: %X\n", float32(b.size)/1024, b.size, b.hash32.Sum32())
	gen.WriteGoFile("tables.go", "language", w.Bytes())
}
コード例 #6
0
func main() {
	gen.Init()

	gen.Repackage("gen_common.go", "common.go", "currency")

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental", "main")
	d.SetSectionFilter("numbers")
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer w.WriteGoFile(*outputFile, "currency")

	fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)

	gen.WriteCLDRVersion(w)
	b := &builder{}
	b.genCurrencies(w, data.Supplemental())
	b.genSymbols(w, data)
}
コード例 #7
0
ファイル: maketables.go プロジェクト: 7ukey/text
func main() {
	gen.Init()

	gen.Repackage("gen_common.go", "common.go", "language")

	w := gen.NewCodeWriter()
	defer w.WriteGoFile("tables.go", "language")

	fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)

	b := newBuilder(w)
	gen.WriteCLDRVersion(w)

	b.parseIndices()
	b.writeType(fromTo{})
	b.writeLanguage()
	b.writeScript()
	b.writeRegion()
	b.writeVariant()
	// TODO: b.writeLocale()
	b.computeRegionGroups()
	b.writeLikelyData()
	b.writeMatchData()
	b.writeRegionInclusionData()
	b.writeParents()
}
コード例 #8
0
ファイル: maketables.go プロジェクト: taras-yurchenko/text
func main() {
	gen.Init()

	rewriteCommon()

	w := gen.NewCodeWriter()
	defer w.WriteGoFile("tables.go", "language")

	b := newBuilder(w)
	fmt.Fprintf(w, version, cldr.Version)

	b.parseIndices()
	b.writeType(fromTo{})
	b.writeLanguage()
	b.writeScript()
	b.writeRegion()
	b.writeVariant()
	// TODO: b.writeLocale()
	b.writeCurrencies()
	b.computeRegionGroups()
	b.writeLikelyData()
	b.writeMatchData()
	b.writeRegionInclusionData()
	b.writeParents()
}
コード例 #9
0
ファイル: gen.go プロジェクト: 7ukey/text
func main() {
	gen.Init()
	gen.Repackage("gen_trieval.go", "trieval.go", "bidi")
	gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi")

	genTables()
}
コード例 #10
0
ファイル: gen.go プロジェクト: ChongFeng/beats
func main() {
	gen.Init()

	r := gen.Open("http://www.w3.org/TR", "w3", "encoding/indexes/encodings.json")
	var groups []group
	if err := json.NewDecoder(r).Decode(&groups); err != nil {
		log.Fatalf("Error reading encodings.json: %v", err)
	}

	w := &bytes.Buffer{}
	fmt.Fprintln(w, "type htmlEncoding byte")
	fmt.Fprintln(w, "const (")
	for i, g := range groups {
		for _, e := range g.Encodings {
			name := consts[e.Name]
			if name == "" {
				log.Fatalf("No const defined for %s.", e.Name)
			}
			if i == 0 {
				fmt.Fprintf(w, "%s htmlEncoding = iota\n", name)
			} else {
				fmt.Fprintf(w, "%s\n", name)
			}
		}
	}
	fmt.Fprintln(w, "numEncodings")
	fmt.Fprint(w, ")\n\n")

	fmt.Fprintln(w, "var canonical = [numEncodings]string{")
	for _, g := range groups {
		for _, e := range g.Encodings {
			fmt.Fprintf(w, "%q,\n", e.Name)
		}
	}
	fmt.Fprint(w, "}\n\n")

	fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{")
	for _, g := range groups {
		for _, e := range g.Encodings {
			for _, l := range e.Labels {
				fmt.Fprintf(w, "%q: %s,\n", l, consts[e.Name])
			}
		}
	}
	fmt.Fprint(w, "}\n\n")

	var tags []string
	fmt.Fprintln(w, "var localeMap = []htmlEncoding{")
	for _, loc := range locales {
		tags = append(tags, loc.tag)
		fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag)
	}
	fmt.Fprint(w, "}\n\n")

	fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " "))

	gen.WriteGoFile("tables.go", "htmlindex", w.Bytes())
}
コード例 #11
0
ファイル: tables_test.go プロジェクト: bigwhite/gocmpp
func TestTables(t *testing.T) {
	if !*long {
		return
	}

	gen.Init()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental", "main")
	d.SetSectionFilter("numbers")
	data, err := d.DecodeZip(r)
	if err != nil {
		t.Fatalf("DecodeZip: %v", err)
	}

	dr, err := cldr.ParseDraft(*draft)
	if err != nil {
		t.Fatalf("filter: %v", err)
	}

	for _, lang := range data.Locales() {
		p := message.NewPrinter(language.MustParse(lang))

		ldml := data.RawLDML(lang)
		if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
			continue
		}
		for _, c := range ldml.Numbers.Currencies.Currency {
			syms := cldr.MakeSlice(&c.Symbol)
			syms.SelectDraft(dr)

			for _, sym := range c.Symbol {
				cur, err := ParseISO(c.Type)
				if err != nil {
					continue
				}
				formatter := Symbol
				switch sym.Alt {
				case "":
				case "narrow":
					formatter = NarrowSymbol
				default:
					continue
				}
				want := sym.Data()
				if got := p.Sprint(formatter(cur)); got != want {
					t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
				}
			}
		}
	}
}
コード例 #12
0
ファイル: gen.go プロジェクト: glerchundi/kube2nginx
func main() {
	gen.Init()
	args = flag.Args()
	if !*verbose {
		// Set vprintf to a no-op.
		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
	}

	// TODO: create temporary cache directory to load files and create and set
	// a "cache" option if the user did not specify the UNICODE_DIR environment
	// variable. This will prevent duplicate downloads and also will enable long
	// tests, which really need to be run after each generated package.

	if gen.UnicodeVersion() != unicode.Version {
		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
			gen.UnicodeVersion,
			unicode.Version)
		// TODO: use collate to compare. Simple comparison will work, though,
		// until Unicode reaches version 10. To avoid circular dependencies, we
		// could use the NumericWeighter without using package collate using a
		// trivial Weighter implementation.
		if gen.UnicodeVersion() < unicode.Version && !*force {
			os.Exit(2)
		}
	}
	var (
		cldr       = generate("unicode/cldr")
		language   = generate("language", cldr)
		internal   = generate("internal", language)
		norm       = generate("unicode/norm")
		rangetable = generate("unicode/rangetable")
		cases      = generate("cases", norm, language, rangetable)
		width      = generate("width")
		bidi       = generate("unicode/bidi", norm, rangetable)
		_          = generate("secure/precis", norm, rangetable, cases, width, bidi)
		_          = generate("encoding/htmlindex", language)
		_          = generate("currency", cldr, language, internal)
		_          = generate("internal/number", cldr, language, internal)
		_          = generate("language/display", cldr, language)
		_          = generate("collate", norm, cldr, language, rangetable)
		_          = generate("search", norm, cldr, language, rangetable)
	)
	all.Wait()

	if hasErrors {
		fmt.Println("FAIL")
		os.Exit(1)
	}
	vprintf("SUCCESS\n")
}
コード例 #13
0
ファイル: gen.go プロジェクト: luizbafilho/fusis
func main() {
	gen.Init()

	names, counts := parse()
	appendRepeatNames(names, counts)
	appendUniqueNames(names, counts)

	table0, table1 := makeTables()

	gen.Repackage("gen_bits.go", "bits.go", "runenames")

	w := gen.NewCodeWriter()
	w.WriteVar("table0", table0)
	w.WriteVar("table1", table1)
	w.WriteConst("data", string(data))
	w.WriteGoFile("tables.go", "runenames")
}
コード例 #14
0
func main() {
	gen.Init()
	loadUnicodeData()
	compactCCC()
	loadCompositionExclusions()
	completeCharFields(FCanonical)
	completeCharFields(FCompatibility)
	computeNonStarterCounts()
	verifyComputed()
	printChars()
	if *test {
		testDerived()
		printTestdata()
	} else {
		makeTables()
	}
}
コード例 #15
0
ファイル: main.go プロジェクト: bom-d-van/csfw
func main() {
	gen.Init()

	// Read the CLDR zip file. Autodownloading if file not found
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("main", "supplemental")
	d.SetSectionFilter("localeDisplayNames", "numbers")
	data, err := d.DecodeZip(r)
	codegen.LogFatal(err, "DecodeZip: %v", err)

	curW := &bytes.Buffer{}
	for _, loc := range data.Locales() {

		if false == codegen.ConfigLocalization.EnabledLocale.Include(loc) {
			continue
		}

		ldml, err := data.LDML(loc)
		codegen.LogFatal(err)
		fmt.Fprintf(os.Stdout, "Generating: %s\n", loc)

		curB := curBuilder{
			w:      curW,
			locale: loc,
			data:   ldml,
		}
		curB.generate()
	}

	tplData := map[string]interface{}{
		"Package":       codegen.ConfigLocalization.Package,
		"CurrencyDicts": curW.String(),
	}

	formatted, err := codegen.GenerateCode(codegen.ConfigLocalization.Package, tplCode, tplData, nil)
	if err != nil {
		codegen.LogFatal(err, "\n%s\n", formatted)
	}

	codegen.LogFatal(ioutil.WriteFile(codegen.ConfigLocalization.OutputFile, formatted, 0600))
}
コード例 #16
0
ファイル: gen.go プロジェクト: Zhoutall/beats
func main() {
	gen.Init()

	versions := getVersions()

	w := &bytes.Buffer{}

	fmt.Fprintf(w, "//go:generate go run gen.go --versions=%s\n\n", strings.Join(versions, ","))
	fmt.Fprintf(w, "import \"unicode\"\n\n")

	vstr := func(s string) string { return strings.Replace(s, ".", "_", -1) }

	fmt.Fprintf(w, "var assigned = map[string]*unicode.RangeTable{\n")
	for _, v := range versions {
		fmt.Fprintf(w, "\t%q: assigned%s,\n", v, vstr(v))
	}
	fmt.Fprintf(w, "}\n\n")

	var size int
	for _, v := range versions {
		assigned := []rune{}

		r := gen.Open("http://www.unicode.org/Public/", "", v+"/ucd/UnicodeData.txt")
		ucd.Parse(r, func(p *ucd.Parser) {
			assigned = append(assigned, p.Rune(0))
		})

		rt := rangetable.New(assigned...)
		sz := int(reflect.TypeOf(unicode.RangeTable{}).Size())
		sz += int(reflect.TypeOf(unicode.Range16{}).Size()) * len(rt.R16)
		sz += int(reflect.TypeOf(unicode.Range32{}).Size()) * len(rt.R32)

		fmt.Fprintf(w, "// size %d bytes (%d KiB)\n", sz, sz/1024)
		fmt.Fprintf(w, "var assigned%s = ", vstr(v))
		print(w, rt)

		size += sz
	}

	fmt.Fprintf(w, "// Total size %d bytes (%d KiB)\n", size, size/1024)

	gen.WriteGoFile("tables.go", "rangetable", w.Bytes())
}
コード例 #17
0
ファイル: gen.go プロジェクト: ChongFeng/beats
func main() {
	gen.Init()
	args = flag.Args()
	if !*verbose {
		// Set vprintf to a no-op.
		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
	}

	if gen.UnicodeVersion() != unicode.Version {
		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
			gen.UnicodeVersion,
			unicode.Version)
		// TODO: use collate to compare. Simple comparison will work, though,
		// until Unicode reaches version 10. To avoid circular dependencies, we
		// could use the NumericWeighter without using package collate using a
		// trivial Weighter implementation.
		if gen.UnicodeVersion() < unicode.Version && !*force {
			os.Exit(2)
		}
	}
	var (
		cldr     = generate("cldr")
		language = generate("language", cldr)
		internal = generate("internal", language)
		norm     = generate("unicode/norm")
		_        = generate("unicode/rangetable")
		_        = generate("width")
		_        = generate("currency", cldr, language, internal)
		_        = generate("display", cldr, language)
		_        = generate("cases", norm)
		_        = generate("collate", norm, cldr, language)
		_        = generate("search", norm, cldr, language)
	)
	all.Wait()

	if hasErrors {
		fmt.Println("FAIL")
		os.Exit(1)
	}
	vprintf("SUCCESS\n")
}
コード例 #18
0
ファイル: tables_test.go プロジェクト: 7ukey/text
func TestTables(t *testing.T) {
	if !*long {
		return
	}

	gen.Init()

	trie := newBidiTrie(0)

	parse("BidiBrackets.txt", func(p *ucd.Parser) {
		r1 := p.Rune(0)
		want := p.Rune(1)

		e, _ := trie.lookupString(string(r1))
		if got := entry(e).reverseBracket(r1); got != want {
			t.Errorf("Reverse(%U) = %U; want %U", r1, got, want)
		}
	})

	done := map[rune]bool{}
	test := func(name string, r rune, want string) {
		e, _ := trie.lookupString(string(r))
		if got := labels[entry(e).class(r)]; got != want {
			t.Errorf("%s:%U: got %s; want %s", name, r, got, want)
		}
		done[r] = true
	}

	// Insert the derived BiDi properties.
	parse("extracted/DerivedBidiClass.txt", func(p *ucd.Parser) {
		r := p.Rune(0)
		test("derived", r, p.String(1))
	})
	visitDefaults(func(r rune, c class) {
		if !done[r] {
			test("default", r, labels[c])
		}
	})

}
コード例 #19
0
ファイル: gen.go プロジェクト: glerchundi/kube2nginx
func main() {
	gen.Init()

	// Load data
	runes := []rune{}
	ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) {
		if p.String(1) == "Default_Ignorable_Code_Point" {
			runes = append(runes, p.Rune(0))
		}
	})
	ucd.Parse(gen.OpenUCDFile("HangulSyllableType.txt"), func(p *ucd.Parser) {
		if p.String(1) == "LVT" {
			runes = append(runes, p.Rune(0))
		}
	})

	disallowedRunes = rangetable.New(runes...)
	assigned = rangetable.Assigned(unicode.Version)

	writeTables()
	gen.Repackage("gen_trieval.go", "trieval.go", "precis")
}
コード例 #20
0
ファイル: gen.go プロジェクト: rdterner/text
func main() {
	gen.Init()

	rewriteCommon()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental")
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer w.WriteGoFile(*outputFile, "currency")

	fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)

	gen.WriteCLDRVersion(w)
	genCurrencies(w, data.Supplemental())
}
コード例 #21
0
ファイル: gen.go プロジェクト: YaSuenag/hsbeat
func main() {
	gen.Init()

	// Load data
	runes := []rune{}
	// PrecisIgnorableProperties: https://tools.ietf.org/html/rfc7564#section-9.13
	ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) {
		if p.String(1) == "Default_Ignorable_Code_Point" {
			runes = append(runes, p.Rune(0))
		}
	})
	ucd.Parse(gen.OpenUCDFile("PropList.txt"), func(p *ucd.Parser) {
		switch p.String(1) {
		case "Noncharacter_Code_Point":
			runes = append(runes, p.Rune(0))
		}
	})
	// OldHangulJamo: https://tools.ietf.org/html/rfc5892#section-2.9
	ucd.Parse(gen.OpenUCDFile("HangulSyllableType.txt"), func(p *ucd.Parser) {
		switch p.String(1) {
		case "L", "V", "T":
			runes = append(runes, p.Rune(0))
		}
	})

	disallowedRunes = rangetable.New(runes...)
	assigned = rangetable.Assigned(unicode.Version)

	// Load category data.
	runeCategory['l'] = latinSmallL
	ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
		const cccVirama = 9
		if p.Int(ucd.CanonicalCombiningClass) == cccVirama {
			setCategory(p.Rune(0), viramaModifier)
		}
	})
	ucd.Parse(gen.OpenUCDFile("Scripts.txt"), func(p *ucd.Parser) {
		switch p.String(1) {
		case "Greek":
			setCategory(p.Rune(0), greek)
		case "Hebrew":
			setCategory(p.Rune(0), hebrew)
		case "Hiragana", "Katakana", "Han":
			setCategory(p.Rune(0), japanese)
		}
	})

	// Set the rule categories associated with exceptions. This overrides any
	// previously set categories. The original categories are manually
	// reintroduced in the categoryTransitions table.
	for r, e := range exceptions {
		if e.cat != 0 {
			runeCategory[r] = e.cat
		}
	}
	cat := map[string]category{
		"L": joiningL,
		"D": joiningD,
		"T": joiningT,

		"R": joiningR,
	}
	ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) {
		switch v := p.String(1); v {
		case "L", "D", "T", "R":
			setCategory(p.Rune(0), cat[v])
		}
	})

	writeTables()
	gen.Repackage("gen_trieval.go", "trieval.go", "precis")
}
コード例 #22
0
ファイル: gen.go プロジェクト: ChongFeng/beats
func main() {
	gen.Init()
	genTables()
	genTablesTest()
	genTrieval()
}
コード例 #23
0
ファイル: gen.go プロジェクト: luizbafilho/fusis
func main() {
	gen.Init()
	genTables()
	gen.Repackage("gen_trieval.go", "trieval.go", "idna")
	gen.Repackage("gen_common.go", "common_test.go", "idna")
}
コード例 #24
0
ファイル: gen_index.go プロジェクト: ChongFeng/beats
func main() {
	gen.Init()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer func() {
		buf := &bytes.Buffer{}

		if _, err = w.WriteGo(buf, "language"); err != nil {
			log.Fatalf("Error formatting file index.go: %v", err)
		}

		// Since we're generating a table for our own package we need to rewrite
		// doing the equivalent of go fmt -r 'language.b -> b'. Using
		// bytes.Replace will do.
		out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
		if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
			log.Fatalf("Could not create file index.go: %v", err)
		}
	}()

	m := map[language.Tag]bool{}
	for _, lang := range data.Locales() {
		if x := data.RawLDML(lang); false ||
			x.LocaleDisplayNames != nil ||
			x.Characters != nil ||
			x.Delimiters != nil ||
			x.Measurement != nil ||
			x.Dates != nil ||
			x.Numbers != nil ||
			x.Units != nil ||
			x.ListPatterns != nil ||
			x.Collations != nil ||
			x.Segmentations != nil ||
			x.Rbnf != nil ||
			x.Annotations != nil ||
			x.Metadata != nil {

			// TODO: support POSIX natively, albeit non-standard.
			tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
			m[tag] = true
		}
	}
	var core, special []language.Tag

	for t := range m {
		if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
			log.Fatalf("Unexpected extension %v in %v", x, t)
		}
		if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
			core = append(core, t)
		} else {
			special = append(special, t)
		}
	}

	w.WriteComment(`
	NumCompactTags is the number of common tags. The maximum tag is
	NumCompactTags-1.`)
	w.WriteConst("NumCompactTags", len(core)+len(special))

	sort.Sort(byAlpha(special))
	w.WriteVar("specialTags", special)

	type coreKey struct {
		base   language.Base
		script language.Script
		region language.Region
	}
	w.WriteType(coreKey{})

	// TODO: order by frequency?
	sort.Sort(byAlpha(core))

	// Size computations are just an estimate.
	w.Size += int(reflect.TypeOf(map[coreKey]uint16{}).Size())
	w.Size += len(core) * int(reflect.TypeOf(coreKey{}).Size()+2) // 2 is for uint16

	fmt.Fprintln(w, "var coreTags = map[coreKey]uint16{")
	fmt.Fprintln(w, "coreKey{}: 0, // und")
	i := len(special) + 1 // Und and special tags already written.
	for _, t := range core {
		if t == language.Und {
			continue
		}
		fmt.Fprint(w.Hash, t, i)
		b, s, r := t.Raw()
		key := fmt.Sprintf("%#v", coreKey{b, s, r})
		key = strings.Replace(key[len("main."):], "language.", "", -1)
		fmt.Fprintf(w, "%s: %d, // %s\n", key, i, t)
		i++
	}
	fmt.Fprintln(w, "}")
}
コード例 #25
0
ファイル: gen_index.go プロジェクト: Codzart/go-ethereum
func main() {
	gen.Init()

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer func() {
		buf := &bytes.Buffer{}

		if _, err = w.WriteGo(buf, "language"); err != nil {
			log.Fatalf("Error formatting file index.go: %v", err)
		}

		// Since we're generating a table for our own package we need to rewrite
		// doing the equivalent of go fmt -r 'language.b -> b'. Using
		// bytes.Replace will do.
		out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
		if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
			log.Fatalf("Could not create file index.go: %v", err)
		}
	}()

	m := map[language.Tag]bool{}
	for _, lang := range data.Locales() {
		// We include all locales unconditionally to be consistent with en_US.
		// We want en_US, even though it has no data associated with it.

		// TODO: put any of the languages for which no data exists at the end
		// of the index. This allows all components based on ICU to use that
		// as the cutoff point.
		// if x := data.RawLDML(lang); false ||
		// 	x.LocaleDisplayNames != nil ||
		// 	x.Characters != nil ||
		// 	x.Delimiters != nil ||
		// 	x.Measurement != nil ||
		// 	x.Dates != nil ||
		// 	x.Numbers != nil ||
		// 	x.Units != nil ||
		// 	x.ListPatterns != nil ||
		// 	x.Collations != nil ||
		// 	x.Segmentations != nil ||
		// 	x.Rbnf != nil ||
		// 	x.Annotations != nil ||
		// 	x.Metadata != nil {

		// TODO: support POSIX natively, albeit non-standard.
		tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
		m[tag] = true
		// }
	}
	// Include locales for plural rules, which uses a different structure.
	for _, plurals := range data.Supplemental().Plurals {
		for _, rules := range plurals.PluralRules {
			for _, lang := range strings.Split(rules.Locales, " ") {
				m[language.Make(lang)] = true
			}
		}
	}

	var core, special []language.Tag

	for t := range m {
		if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
			log.Fatalf("Unexpected extension %v in %v", x, t)
		}
		if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
			core = append(core, t)
		} else {
			special = append(special, t)
		}
	}

	w.WriteComment(`
	NumCompactTags is the number of common tags. The maximum tag is
	NumCompactTags-1.`)
	w.WriteConst("NumCompactTags", len(core)+len(special))

	sort.Sort(byAlpha(special))
	w.WriteVar("specialTags", special)

	// TODO: order by frequency?
	sort.Sort(byAlpha(core))

	// Size computations are just an estimate.
	w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size())
	w.Size += len(core) * 6 // size of uint32 and uint16

	fmt.Fprintln(w)
	fmt.Fprintln(w, "var coreTags = map[uint32]uint16{")
	fmt.Fprintln(w, "0x0: 0, // und")
	i := len(special) + 1 // Und and special tags already written.
	for _, t := range core {
		if t == language.Und {
			continue
		}
		fmt.Fprint(w.Hash, t, i)
		b, s, r := t.Raw()
		fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n",
			getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number
			getIndex(s, 2),
			getIndex(r, 3),
			i, t)
		i++
	}
	fmt.Fprintln(w, "}")
}
コード例 #26
0
ファイル: gen.go プロジェクト: glerchundi/etcd2-bootstrapper
func main() {
	gen.Init()
	genTables()
	genTablesTest()
	gen.Repackage("gen_trieval.go", "trieval.go", "cases")
}
コード例 #27
0
ファイル: gen.go プロジェクト: luizbafilho/fusis
func main() {
	gen.Init()
	args = flag.Args()
	if !*verbose {
		// Set vprintf to a no-op.
		vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
	}

	// TODO: create temporary cache directory to load files and create and set
	// a "cache" option if the user did not specify the UNICODE_DIR environment
	// variable. This will prevent duplicate downloads and also will enable long
	// tests, which really need to be run after each generated package.

	updateCore := *doCore
	if gen.UnicodeVersion() != unicode.Version {
		fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
			gen.UnicodeVersion(),
			unicode.Version)
		// TODO: use collate to compare. Simple comparison will work, though,
		// until Unicode reaches version 10. To avoid circular dependencies, we
		// could use the NumericWeighter without using package collate using a
		// trivial Weighter implementation.
		if gen.UnicodeVersion() < unicode.Version && !*force {
			os.Exit(2)
		}
		updateCore = true
	}

	var unicode = &dependency{}
	if updateCore {
		fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
		unicode = generate("unicode")

		// Test some users of the unicode packages, especially the ones that
		// keep a mirrored table. These may need to be corrected by hand.
		generate("regexp", unicode)
		generate("strconv", unicode) // mimics Unicode table
		generate("strings", unicode)
		generate("testing", unicode) // mimics Unicode table
	}

	var (
		cldr       = generate("./unicode/cldr", unicode)
		language   = generate("./language", cldr)
		internal   = generate("./internal", unicode, language)
		norm       = generate("./unicode/norm", unicode)
		rangetable = generate("./unicode/rangetable", unicode)
		cases      = generate("./cases", unicode, norm, language, rangetable)
		width      = generate("./width", unicode)
		bidi       = generate("./unicode/bidi", unicode, norm, rangetable)
		_          = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
		_          = generate("./encoding/htmlindex", unicode, language)
		_          = generate("./currency", unicode, cldr, language, internal)
		_          = generate("./internal/number", unicode, cldr, language, internal)
		_          = generate("./language/display", unicode, cldr, language, internal)
		_          = generate("./collate", unicode, norm, cldr, language, rangetable)
		_          = generate("./search", unicode, norm, cldr, language, rangetable)
	)

	if updateCore {
		copyVendored()
		generate("vendor/golang_org/x/net/idna", unicode, norm, width, cases)
	}
	all.Wait()

	if hasErrors {
		fmt.Println("FAIL")
		os.Exit(1)
	}
	vprintf("SUCCESS\n")
}