func main() {
gen.Init()
genTables()
genTests()
repackage("gen_trieval.go", "trieval.go")
repackage("gen_common.go", "common_test.go")
}
func main() {
gen.Init()
const pkg = "number"
gen.Repackage("gen_common.go", "common.go", pkg)
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers", "numberingSystem")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile(*outputFile, pkg)
fmt.Fprintln(w, `import "golang.org/x/text/internal/stringset"`)
gen.WriteCLDRVersion(w)
genNumSystem(w, data)
genSymbols(w, data)
}
func main() {
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("main", "supplemental")
d.SetSectionFilter("localeDisplayNames")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := &bytes.Buffer{}
gen.WriteCLDRVersion(w)
b := builder{
w: w,
data: data,
group: make(map[string]*group),
}
b.generate()
gen.WriteGoFile(*outputFile, "display", w.Bytes())
}
func main() {
gen.Init()
b := build.NewBuilder()
parseUCA(b)
if tables.contains("chars") {
parseMain()
}
parseCollation(b)
c, err := b.Build()
failOnError(err)
if *test {
testCollator(collate.NewFromTable(c))
} else {
w := &bytes.Buffer{}
gen.WriteUnicodeVersion(w)
gen.WriteCLDRVersion(w)
if tables.contains("collate") {
_, err = b.Print(w)
failOnError(err)
}
if tables.contains("chars") {
printExemplarCharacters(w)
}
gen.WriteGoFile("tables.go", *pkg, w.Bytes())
}
}
func main() {
gen.Init()
rewriteCommon()
w := &bytes.Buffer{}
b := newBuilder(w)
fmt.Fprintf(w, version, cldr.Version)
b.parseIndices()
b.writeType(fromTo{})
b.writeLanguage()
b.writeScript()
b.writeRegion()
b.writeVariant()
// TODO: b.writeLocale()
b.writeCurrencies()
b.computeRegionGroups()
b.writeLikelyData()
b.writeMatchData()
b.writeRegionInclusionData()
b.writeParents()
fmt.Fprintf(w, "\n// Size: %.1fK (%d bytes); Check: %X\n", float32(b.size)/1024, b.size, b.hash32.Sum32())
gen.WriteGoFile("tables.go", "language", w.Bytes())
}
func main() {
gen.Init()
gen.Repackage("gen_common.go", "common.go", "currency")
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile(*outputFile, "currency")
fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)
gen.WriteCLDRVersion(w)
b := &builder{}
b.genCurrencies(w, data.Supplemental())
b.genSymbols(w, data)
}
func main() {
gen.Init()
gen.Repackage("gen_common.go", "common.go", "language")
w := gen.NewCodeWriter()
defer w.WriteGoFile("tables.go", "language")
fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)
b := newBuilder(w)
gen.WriteCLDRVersion(w)
b.parseIndices()
b.writeType(fromTo{})
b.writeLanguage()
b.writeScript()
b.writeRegion()
b.writeVariant()
// TODO: b.writeLocale()
b.computeRegionGroups()
b.writeLikelyData()
b.writeMatchData()
b.writeRegionInclusionData()
b.writeParents()
}
func main() {
gen.Init()
rewriteCommon()
w := gen.NewCodeWriter()
defer w.WriteGoFile("tables.go", "language")
b := newBuilder(w)
fmt.Fprintf(w, version, cldr.Version)
b.parseIndices()
b.writeType(fromTo{})
b.writeLanguage()
b.writeScript()
b.writeRegion()
b.writeVariant()
// TODO: b.writeLocale()
b.writeCurrencies()
b.computeRegionGroups()
b.writeLikelyData()
b.writeMatchData()
b.writeRegionInclusionData()
b.writeParents()
}
func main() {
gen.Init()
gen.Repackage("gen_trieval.go", "trieval.go", "bidi")
gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi")
genTables()
}
func main() {
gen.Init()
r := gen.Open("http://www.w3.org/TR", "w3", "encoding/indexes/encodings.json")
var groups []group
if err := json.NewDecoder(r).Decode(&groups); err != nil {
log.Fatalf("Error reading encodings.json: %v", err)
}
w := &bytes.Buffer{}
fmt.Fprintln(w, "type htmlEncoding byte")
fmt.Fprintln(w, "const (")
for i, g := range groups {
for _, e := range g.Encodings {
name := consts[e.Name]
if name == "" {
log.Fatalf("No const defined for %s.", e.Name)
}
if i == 0 {
fmt.Fprintf(w, "%s htmlEncoding = iota\n", name)
} else {
fmt.Fprintf(w, "%s\n", name)
}
}
}
fmt.Fprintln(w, "numEncodings")
fmt.Fprint(w, ")\n\n")
fmt.Fprintln(w, "var canonical = [numEncodings]string{")
for _, g := range groups {
for _, e := range g.Encodings {
fmt.Fprintf(w, "%q,\n", e.Name)
}
}
fmt.Fprint(w, "}\n\n")
fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{")
for _, g := range groups {
for _, e := range g.Encodings {
for _, l := range e.Labels {
fmt.Fprintf(w, "%q: %s,\n", l, consts[e.Name])
}
}
}
fmt.Fprint(w, "}\n\n")
var tags []string
fmt.Fprintln(w, "var localeMap = []htmlEncoding{")
for _, loc := range locales {
tags = append(tags, loc.tag)
fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag)
}
fmt.Fprint(w, "}\n\n")
fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " "))
gen.WriteGoFile("tables.go", "htmlindex", w.Bytes())
}
func TestTables(t *testing.T) {
if !*long {
return
}
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
dr, err := cldr.ParseDraft(*draft)
if err != nil {
t.Fatalf("filter: %v", err)
}
for _, lang := range data.Locales() {
p := message.NewPrinter(language.MustParse(lang))
ldml := data.RawLDML(lang)
if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
continue
}
for _, c := range ldml.Numbers.Currencies.Currency {
syms := cldr.MakeSlice(&c.Symbol)
syms.SelectDraft(dr)
for _, sym := range c.Symbol {
cur, err := ParseISO(c.Type)
if err != nil {
continue
}
formatter := Symbol
switch sym.Alt {
case "":
case "narrow":
formatter = NarrowSymbol
default:
continue
}
want := sym.Data()
if got := p.Sprint(formatter(cur)); got != want {
t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
}
}
}
}
}
func main() {
gen.Init()
args = flag.Args()
if !*verbose {
// Set vprintf to a no-op.
vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
}
// TODO: create temporary cache directory to load files and create and set
// a "cache" option if the user did not specify the UNICODE_DIR environment
// variable. This will prevent duplicate downloads and also will enable long
// tests, which really need to be run after each generated package.
if gen.UnicodeVersion() != unicode.Version {
fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
gen.UnicodeVersion,
unicode.Version)
// TODO: use collate to compare. Simple comparison will work, though,
// until Unicode reaches version 10. To avoid circular dependencies, we
// could use the NumericWeighter without using package collate using a
// trivial Weighter implementation.
if gen.UnicodeVersion() < unicode.Version && !*force {
os.Exit(2)
}
}
var (
cldr = generate("unicode/cldr")
language = generate("language", cldr)
internal = generate("internal", language)
norm = generate("unicode/norm")
rangetable = generate("unicode/rangetable")
cases = generate("cases", norm, language, rangetable)
width = generate("width")
bidi = generate("unicode/bidi", norm, rangetable)
_ = generate("secure/precis", norm, rangetable, cases, width, bidi)
_ = generate("encoding/htmlindex", language)
_ = generate("currency", cldr, language, internal)
_ = generate("internal/number", cldr, language, internal)
_ = generate("language/display", cldr, language)
_ = generate("collate", norm, cldr, language, rangetable)
_ = generate("search", norm, cldr, language, rangetable)
)
all.Wait()
if hasErrors {
fmt.Println("FAIL")
os.Exit(1)
}
vprintf("SUCCESS\n")
}
func main() {
gen.Init()
names, counts := parse()
appendRepeatNames(names, counts)
appendUniqueNames(names, counts)
table0, table1 := makeTables()
gen.Repackage("gen_bits.go", "bits.go", "runenames")
w := gen.NewCodeWriter()
w.WriteVar("table0", table0)
w.WriteVar("table1", table1)
w.WriteConst("data", string(data))
w.WriteGoFile("tables.go", "runenames")
}
func main() {
gen.Init()
loadUnicodeData()
compactCCC()
loadCompositionExclusions()
completeCharFields(FCanonical)
completeCharFields(FCompatibility)
computeNonStarterCounts()
verifyComputed()
printChars()
if *test {
testDerived()
printTestdata()
} else {
makeTables()
}
}
func main() {
gen.Init()
// Read the CLDR zip file. Autodownloading if file not found
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("main", "supplemental")
d.SetSectionFilter("localeDisplayNames", "numbers")
data, err := d.DecodeZip(r)
codegen.LogFatal(err, "DecodeZip: %v", err)
curW := &bytes.Buffer{}
for _, loc := range data.Locales() {
if false == codegen.ConfigLocalization.EnabledLocale.Include(loc) {
continue
}
ldml, err := data.LDML(loc)
codegen.LogFatal(err)
fmt.Fprintf(os.Stdout, "Generating: %s\n", loc)
curB := curBuilder{
w: curW,
locale: loc,
data: ldml,
}
curB.generate()
}
tplData := map[string]interface{}{
"Package": codegen.ConfigLocalization.Package,
"CurrencyDicts": curW.String(),
}
formatted, err := codegen.GenerateCode(codegen.ConfigLocalization.Package, tplCode, tplData, nil)
if err != nil {
codegen.LogFatal(err, "\n%s\n", formatted)
}
codegen.LogFatal(ioutil.WriteFile(codegen.ConfigLocalization.OutputFile, formatted, 0600))
}
func main() {
gen.Init()
versions := getVersions()
w := &bytes.Buffer{}
fmt.Fprintf(w, "//go:generate go run gen.go --versions=%s\n\n", strings.Join(versions, ","))
fmt.Fprintf(w, "import \"unicode\"\n\n")
vstr := func(s string) string { return strings.Replace(s, ".", "_", -1) }
fmt.Fprintf(w, "var assigned = map[string]*unicode.RangeTable{\n")
for _, v := range versions {
fmt.Fprintf(w, "\t%q: assigned%s,\n", v, vstr(v))
}
fmt.Fprintf(w, "}\n\n")
var size int
for _, v := range versions {
assigned := []rune{}
r := gen.Open("http://www.unicode.org/Public/", "", v+"/ucd/UnicodeData.txt")
ucd.Parse(r, func(p *ucd.Parser) {
assigned = append(assigned, p.Rune(0))
})
rt := rangetable.New(assigned...)
sz := int(reflect.TypeOf(unicode.RangeTable{}).Size())
sz += int(reflect.TypeOf(unicode.Range16{}).Size()) * len(rt.R16)
sz += int(reflect.TypeOf(unicode.Range32{}).Size()) * len(rt.R32)
fmt.Fprintf(w, "// size %d bytes (%d KiB)\n", sz, sz/1024)
fmt.Fprintf(w, "var assigned%s = ", vstr(v))
print(w, rt)
size += sz
}
fmt.Fprintf(w, "// Total size %d bytes (%d KiB)\n", size, size/1024)
gen.WriteGoFile("tables.go", "rangetable", w.Bytes())
}
func main() {
gen.Init()
args = flag.Args()
if !*verbose {
// Set vprintf to a no-op.
vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
}
if gen.UnicodeVersion() != unicode.Version {
fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
gen.UnicodeVersion,
unicode.Version)
// TODO: use collate to compare. Simple comparison will work, though,
// until Unicode reaches version 10. To avoid circular dependencies, we
// could use the NumericWeighter without using package collate using a
// trivial Weighter implementation.
if gen.UnicodeVersion() < unicode.Version && !*force {
os.Exit(2)
}
}
var (
cldr = generate("cldr")
language = generate("language", cldr)
internal = generate("internal", language)
norm = generate("unicode/norm")
_ = generate("unicode/rangetable")
_ = generate("width")
_ = generate("currency", cldr, language, internal)
_ = generate("display", cldr, language)
_ = generate("cases", norm)
_ = generate("collate", norm, cldr, language)
_ = generate("search", norm, cldr, language)
)
all.Wait()
if hasErrors {
fmt.Println("FAIL")
os.Exit(1)
}
vprintf("SUCCESS\n")
}
func TestTables(t *testing.T) {
if !*long {
return
}
gen.Init()
trie := newBidiTrie(0)
parse("BidiBrackets.txt", func(p *ucd.Parser) {
r1 := p.Rune(0)
want := p.Rune(1)
e, _ := trie.lookupString(string(r1))
if got := entry(e).reverseBracket(r1); got != want {
t.Errorf("Reverse(%U) = %U; want %U", r1, got, want)
}
})
done := map[rune]bool{}
test := func(name string, r rune, want string) {
e, _ := trie.lookupString(string(r))
if got := labels[entry(e).class(r)]; got != want {
t.Errorf("%s:%U: got %s; want %s", name, r, got, want)
}
done[r] = true
}
// Insert the derived BiDi properties.
parse("extracted/DerivedBidiClass.txt", func(p *ucd.Parser) {
r := p.Rune(0)
test("derived", r, p.String(1))
})
visitDefaults(func(r rune, c class) {
if !done[r] {
test("default", r, labels[c])
}
})
}
func main() {
gen.Init()
// Load data
runes := []rune{}
ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) {
if p.String(1) == "Default_Ignorable_Code_Point" {
runes = append(runes, p.Rune(0))
}
})
ucd.Parse(gen.OpenUCDFile("HangulSyllableType.txt"), func(p *ucd.Parser) {
if p.String(1) == "LVT" {
runes = append(runes, p.Rune(0))
}
})
disallowedRunes = rangetable.New(runes...)
assigned = rangetable.Assigned(unicode.Version)
writeTables()
gen.Repackage("gen_trieval.go", "trieval.go", "precis")
}
func main() {
gen.Init()
rewriteCommon()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile(*outputFile, "currency")
fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)
gen.WriteCLDRVersion(w)
genCurrencies(w, data.Supplemental())
}
func main() {
gen.Init()
// Load data
runes := []rune{}
// PrecisIgnorableProperties: https://tools.ietf.org/html/rfc7564#section-9.13
ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) {
if p.String(1) == "Default_Ignorable_Code_Point" {
runes = append(runes, p.Rune(0))
}
})
ucd.Parse(gen.OpenUCDFile("PropList.txt"), func(p *ucd.Parser) {
switch p.String(1) {
case "Noncharacter_Code_Point":
runes = append(runes, p.Rune(0))
}
})
// OldHangulJamo: https://tools.ietf.org/html/rfc5892#section-2.9
ucd.Parse(gen.OpenUCDFile("HangulSyllableType.txt"), func(p *ucd.Parser) {
switch p.String(1) {
case "L", "V", "T":
runes = append(runes, p.Rune(0))
}
})
disallowedRunes = rangetable.New(runes...)
assigned = rangetable.Assigned(unicode.Version)
// Load category data.
runeCategory['l'] = latinSmallL
ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
const cccVirama = 9
if p.Int(ucd.CanonicalCombiningClass) == cccVirama {
setCategory(p.Rune(0), viramaModifier)
}
})
ucd.Parse(gen.OpenUCDFile("Scripts.txt"), func(p *ucd.Parser) {
switch p.String(1) {
case "Greek":
setCategory(p.Rune(0), greek)
case "Hebrew":
setCategory(p.Rune(0), hebrew)
case "Hiragana", "Katakana", "Han":
setCategory(p.Rune(0), japanese)
}
})
// Set the rule categories associated with exceptions. This overrides any
// previously set categories. The original categories are manually
// reintroduced in the categoryTransitions table.
for r, e := range exceptions {
if e.cat != 0 {
runeCategory[r] = e.cat
}
}
cat := map[string]category{
"L": joiningL,
"D": joiningD,
"T": joiningT,
"R": joiningR,
}
ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) {
switch v := p.String(1); v {
case "L", "D", "T", "R":
setCategory(p.Rune(0), cat[v])
}
})
writeTables()
gen.Repackage("gen_trieval.go", "trieval.go", "precis")
}
func main() {
gen.Init()
genTables()
genTablesTest()
genTrieval()
}
func main() {
gen.Init()
genTables()
gen.Repackage("gen_trieval.go", "trieval.go", "idna")
gen.Repackage("gen_common.go", "common_test.go", "idna")
}
func main() {
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer func() {
buf := &bytes.Buffer{}
if _, err = w.WriteGo(buf, "language"); err != nil {
log.Fatalf("Error formatting file index.go: %v", err)
}
// Since we're generating a table for our own package we need to rewrite
// doing the equivalent of go fmt -r 'language.b -> b'. Using
// bytes.Replace will do.
out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
log.Fatalf("Could not create file index.go: %v", err)
}
}()
m := map[language.Tag]bool{}
for _, lang := range data.Locales() {
if x := data.RawLDML(lang); false ||
x.LocaleDisplayNames != nil ||
x.Characters != nil ||
x.Delimiters != nil ||
x.Measurement != nil ||
x.Dates != nil ||
x.Numbers != nil ||
x.Units != nil ||
x.ListPatterns != nil ||
x.Collations != nil ||
x.Segmentations != nil ||
x.Rbnf != nil ||
x.Annotations != nil ||
x.Metadata != nil {
// TODO: support POSIX natively, albeit non-standard.
tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
m[tag] = true
}
}
var core, special []language.Tag
for t := range m {
if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
log.Fatalf("Unexpected extension %v in %v", x, t)
}
if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
core = append(core, t)
} else {
special = append(special, t)
}
}
w.WriteComment(`
NumCompactTags is the number of common tags. The maximum tag is
NumCompactTags-1.`)
w.WriteConst("NumCompactTags", len(core)+len(special))
sort.Sort(byAlpha(special))
w.WriteVar("specialTags", special)
type coreKey struct {
base language.Base
script language.Script
region language.Region
}
w.WriteType(coreKey{})
// TODO: order by frequency?
sort.Sort(byAlpha(core))
// Size computations are just an estimate.
w.Size += int(reflect.TypeOf(map[coreKey]uint16{}).Size())
w.Size += len(core) * int(reflect.TypeOf(coreKey{}).Size()+2) // 2 is for uint16
fmt.Fprintln(w, "var coreTags = map[coreKey]uint16{")
fmt.Fprintln(w, "coreKey{}: 0, // und")
i := len(special) + 1 // Und and special tags already written.
for _, t := range core {
if t == language.Und {
continue
}
fmt.Fprint(w.Hash, t, i)
b, s, r := t.Raw()
key := fmt.Sprintf("%#v", coreKey{b, s, r})
key = strings.Replace(key[len("main."):], "language.", "", -1)
fmt.Fprintf(w, "%s: %d, // %s\n", key, i, t)
i++
}
fmt.Fprintln(w, "}")
}
func main() {
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer func() {
buf := &bytes.Buffer{}
if _, err = w.WriteGo(buf, "language"); err != nil {
log.Fatalf("Error formatting file index.go: %v", err)
}
// Since we're generating a table for our own package we need to rewrite
// doing the equivalent of go fmt -r 'language.b -> b'. Using
// bytes.Replace will do.
out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
log.Fatalf("Could not create file index.go: %v", err)
}
}()
m := map[language.Tag]bool{}
for _, lang := range data.Locales() {
// We include all locales unconditionally to be consistent with en_US.
// We want en_US, even though it has no data associated with it.
// TODO: put any of the languages for which no data exists at the end
// of the index. This allows all components based on ICU to use that
// as the cutoff point.
// if x := data.RawLDML(lang); false ||
// x.LocaleDisplayNames != nil ||
// x.Characters != nil ||
// x.Delimiters != nil ||
// x.Measurement != nil ||
// x.Dates != nil ||
// x.Numbers != nil ||
// x.Units != nil ||
// x.ListPatterns != nil ||
// x.Collations != nil ||
// x.Segmentations != nil ||
// x.Rbnf != nil ||
// x.Annotations != nil ||
// x.Metadata != nil {
// TODO: support POSIX natively, albeit non-standard.
tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
m[tag] = true
// }
}
// Include locales for plural rules, which uses a different structure.
for _, plurals := range data.Supplemental().Plurals {
for _, rules := range plurals.PluralRules {
for _, lang := range strings.Split(rules.Locales, " ") {
m[language.Make(lang)] = true
}
}
}
var core, special []language.Tag
for t := range m {
if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
log.Fatalf("Unexpected extension %v in %v", x, t)
}
if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
core = append(core, t)
} else {
special = append(special, t)
}
}
w.WriteComment(`
NumCompactTags is the number of common tags. The maximum tag is
NumCompactTags-1.`)
w.WriteConst("NumCompactTags", len(core)+len(special))
sort.Sort(byAlpha(special))
w.WriteVar("specialTags", special)
// TODO: order by frequency?
sort.Sort(byAlpha(core))
// Size computations are just an estimate.
w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size())
w.Size += len(core) * 6 // size of uint32 and uint16
fmt.Fprintln(w)
fmt.Fprintln(w, "var coreTags = map[uint32]uint16{")
fmt.Fprintln(w, "0x0: 0, // und")
i := len(special) + 1 // Und and special tags already written.
for _, t := range core {
if t == language.Und {
continue
}
fmt.Fprint(w.Hash, t, i)
b, s, r := t.Raw()
fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n",
getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number
getIndex(s, 2),
getIndex(r, 3),
i, t)
i++
}
fmt.Fprintln(w, "}")
}
func main() {
gen.Init()
genTables()
genTablesTest()
gen.Repackage("gen_trieval.go", "trieval.go", "cases")
}
func main() {
gen.Init()
args = flag.Args()
if !*verbose {
// Set vprintf to a no-op.
vprintf = func(string, ...interface{}) (int, error) { return 0, nil }
}
// TODO: create temporary cache directory to load files and create and set
// a "cache" option if the user did not specify the UNICODE_DIR environment
// variable. This will prevent duplicate downloads and also will enable long
// tests, which really need to be run after each generated package.
updateCore := *doCore
if gen.UnicodeVersion() != unicode.Version {
fmt.Printf("Requested Unicode version %s; core unicode version is %s.\n",
gen.UnicodeVersion(),
unicode.Version)
// TODO: use collate to compare. Simple comparison will work, though,
// until Unicode reaches version 10. To avoid circular dependencies, we
// could use the NumericWeighter without using package collate using a
// trivial Weighter implementation.
if gen.UnicodeVersion() < unicode.Version && !*force {
os.Exit(2)
}
updateCore = true
}
var unicode = &dependency{}
if updateCore {
fmt.Printf("Updating core to version %s...\n", gen.UnicodeVersion())
unicode = generate("unicode")
// Test some users of the unicode packages, especially the ones that
// keep a mirrored table. These may need to be corrected by hand.
generate("regexp", unicode)
generate("strconv", unicode) // mimics Unicode table
generate("strings", unicode)
generate("testing", unicode) // mimics Unicode table
}
var (
cldr = generate("./unicode/cldr", unicode)
language = generate("./language", cldr)
internal = generate("./internal", unicode, language)
norm = generate("./unicode/norm", unicode)
rangetable = generate("./unicode/rangetable", unicode)
cases = generate("./cases", unicode, norm, language, rangetable)
width = generate("./width", unicode)
bidi = generate("./unicode/bidi", unicode, norm, rangetable)
_ = generate("./secure/precis", unicode, norm, rangetable, cases, width, bidi)
_ = generate("./encoding/htmlindex", unicode, language)
_ = generate("./currency", unicode, cldr, language, internal)
_ = generate("./internal/number", unicode, cldr, language, internal)
_ = generate("./language/display", unicode, cldr, language, internal)
_ = generate("./collate", unicode, norm, cldr, language, rangetable)
_ = generate("./search", unicode, norm, cldr, language, rangetable)
)
if updateCore {
copyVendored()
generate("vendor/golang_org/x/net/idna", unicode, norm, width, cases)
}
all.Wait()
if hasErrors {
fmt.Println("FAIL")
os.Exit(1)
}
vprintf("SUCCESS\n")
}
