func main() {
gen.Init()
const pkg = "number"
gen.Repackage("gen_common.go", "common.go", pkg)
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers", "numberingSystem")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile(*outputFile, pkg)
fmt.Fprintln(w, `import "golang.org/x/text/internal/stringset"`)
gen.WriteCLDRVersion(w)
genNumSystem(w, data)
genSymbols(w, data)
}
func TestNumberSystems(t *testing.T) {
testtext.SkipIfNotLong(t)
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental")
d.SetSectionFilter("numberingSystem")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
n := systemMap[ns.Id]
if int(n) >= len(numSysData) {
continue
}
info := InfoFromLangID(0, ns.Id)
val := '0'
for _, rWant := range ns.Digits {
if rGot := info.Digit(val); rGot != rWant {
t.Errorf("%s:%d: got %U; want %U", ns.Id, val, rGot, rWant)
}
val++
}
}
}
func main() {
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("main", "supplemental")
d.SetSectionFilter("localeDisplayNames")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := &bytes.Buffer{}
gen.WriteCLDRVersion(w)
b := builder{
w: w,
data: data,
group: make(map[string]*group),
}
b.generate()
gen.WriteGoFile(*outputFile, "display", w.Bytes())
}
func main() {
gen.Init()
gen.Repackage("gen_common.go", "common.go", "currency")
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile(*outputFile, "currency")
fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)
gen.WriteCLDRVersion(w)
b := &builder{}
b.genCurrencies(w, data.Supplemental())
b.genSymbols(w, data)
}
func openArchive() *zip.Reader {
f := gen.OpenCLDRCoreZip()
buffer, err := ioutil.ReadAll(f)
f.Close()
failOnError(err)
archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
failOnError(err)
return archive
}
func TestTables(t *testing.T) {
if !*long {
return
}
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
dr, err := cldr.ParseDraft(*draft)
if err != nil {
t.Fatalf("filter: %v", err)
}
for _, lang := range data.Locales() {
p := message.NewPrinter(language.MustParse(lang))
ldml := data.RawLDML(lang)
if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
continue
}
for _, c := range ldml.Numbers.Currencies.Currency {
syms := cldr.MakeSlice(&c.Symbol)
syms.SelectDraft(dr)
for _, sym := range c.Symbol {
cur, err := ParseISO(c.Type)
if err != nil {
continue
}
formatter := Symbol
switch sym.Alt {
case "":
case "narrow":
formatter = NarrowSymbol
default:
continue
}
want := sym.Data()
if got := p.Sprint(formatter(cur)); got != want {
t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
}
}
}
}
}
func newBuilder(w *gen.CodeWriter) *builder {
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
data, err := d.DecodeZip(r)
failOnError(err)
b := builder{
w: w,
hw: io.MultiWriter(w, w.Hash),
data: data,
supp: data.Supplemental(),
}
b.parseRegistry()
return &b
}
func newBuilder(w io.Writer) *builder {
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental")
data, err := d.DecodeZip(r)
failOnError(err)
b := builder{
out: w,
data: data,
supp: data.Supplemental(),
hash32: fnv.New32(),
}
b.w = io.MultiWriter(b.out, b.hash32)
b.parseRegistry()
return &b
}
func main() {
gen.Init()
// Read the CLDR zip file. Autodownloading if file not found
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("main", "supplemental")
d.SetSectionFilter("localeDisplayNames", "numbers")
data, err := d.DecodeZip(r)
codegen.LogFatal(err, "DecodeZip: %v", err)
curW := &bytes.Buffer{}
for _, loc := range data.Locales() {
if false == codegen.ConfigLocalization.EnabledLocale.Include(loc) {
continue
}
ldml, err := data.LDML(loc)
codegen.LogFatal(err)
fmt.Fprintf(os.Stdout, "Generating: %s\n", loc)
curB := curBuilder{
w: curW,
locale: loc,
data: ldml,
}
curB.generate()
}
tplData := map[string]interface{}{
"Package": codegen.ConfigLocalization.Package,
"CurrencyDicts": curW.String(),
}
formatted, err := codegen.GenerateCode(codegen.ConfigLocalization.Package, tplCode, tplData, nil)
if err != nil {
codegen.LogFatal(err, "\n%s\n", formatted)
}
codegen.LogFatal(ioutil.WriteFile(codegen.ConfigLocalization.OutputFile, formatted, 0600))
}
func main() {
flag.Parse()
r := gen.OpenCLDRCoreZip()
buffer, err := ioutil.ReadAll(r)
if err != nil {
log.Fatal("Could not read zip file")
}
r.Close()
z, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
if err != nil {
log.Fatalf("Could not read zip archive: %v", err)
}
var buf bytes.Buffer
version := gen.CLDRVersion()
for _, dtd := range files {
for _, f := range z.File {
if strings.HasSuffix(f.Name, dtd.file+".dtd") {
r, err := f.Open()
failOnError(err)
b := makeBuilder(&buf, dtd)
b.parseDTD(r)
b.resolve(b.index[dtd.top[0]])
b.write()
if b.version != "" && version != b.version {
println(f.Name)
log.Fatalf("main: inconsistent versions: found %s; want %s", b.version, version)
}
break
}
}
}
fmt.Fprintln(&buf, "// Version is the version of CLDR from which the XML definitions are generated.")
fmt.Fprintf(&buf, "const Version = %q\n", version)
gen.WriteGoFile(*outputFile, "cldr", buf.Bytes())
}
func main() {
gen.Init()
rewriteCommon()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental")
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile(*outputFile, "currency")
fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)
gen.WriteCLDRVersion(w)
genCurrencies(w, data.Supplemental())
}
func main() {
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer w.WriteGoFile("tables.go", "internal")
// Create parents table.
parents := make([]uint16, language.NumCompactTags)
for _, loc := range data.Locales() {
tag := language.MustParse(loc)
index, ok := language.CompactIndex(tag)
if !ok {
continue
}
parentIndex := 0 // und
for p := tag.Parent(); p != language.Und; p = p.Parent() {
if x, ok := language.CompactIndex(p); ok {
parentIndex = x
break
}
}
parents[index] = uint16(parentIndex)
}
w.WriteComment(`
Parent maps a compact index of a tag to the compact index of the parent of
this tag.`)
w.WriteVar("Parent", parents)
}
func decodeCLDR(d *cldr.Decoder) *cldr.CLDR {
r := gen.OpenCLDRCoreZip()
data, err := d.DecodeZip(r)
failOnError(err)
return data
}
func TestSymbols(t *testing.T) {
testtext.SkipIfNotLong(t)
draft, err := cldr.ParseDraft(*draft)
if err != nil {
log.Fatalf("invalid draft level: %v", err)
}
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
for _, lang := range data.Locales() {
ldml := data.RawLDML(lang)
if ldml.Numbers == nil {
continue
}
langIndex, ok := language.CompactIndex(language.MustParse(lang))
if !ok {
t.Fatalf("No compact index for language %s", lang)
}
syms := cldr.MakeSlice(&ldml.Numbers.Symbols)
syms.SelectDraft(draft)
for _, sym := range ldml.Numbers.Symbols {
if sym.NumberSystem == "" {
continue
}
testCases := []struct {
name string
st SymbolType
x interface{}
}{
{"Decimal", SymDecimal, sym.Decimal},
{"Group", SymGroup, sym.Group},
{"List", SymList, sym.List},
{"PercentSign", SymPercentSign, sym.PercentSign},
{"PlusSign", SymPlusSign, sym.PlusSign},
{"MinusSign", SymMinusSign, sym.MinusSign},
{"Exponential", SymExponential, sym.Exponential},
{"SuperscriptingExponent", SymSuperscriptingExponent, sym.SuperscriptingExponent},
{"PerMille", SymPerMille, sym.PerMille},
{"Infinity", SymInfinity, sym.Infinity},
{"NaN", SymNan, sym.Nan},
{"TimeSeparator", SymTimeSeparator, sym.TimeSeparator},
}
info := InfoFromLangID(langIndex, sym.NumberSystem)
for _, tc := range testCases {
// Extract the wanted value.
v := reflect.ValueOf(tc.x)
if v.Len() == 0 {
return
}
if v.Len() > 1 {
t.Fatalf("Multiple values of %q within single symbol not supported.", tc.name)
}
want := v.Index(0).MethodByName("Data").Call(nil)[0].String()
got := info.Symbol(tc.st)
if got != want {
t.Errorf("%s:%s:%s: got %q; want %q", lang, sym.NumberSystem, tc.name, got, want)
}
}
}
}
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
dr, err := cldr.ParseDraft(*draft)
if err != nil {
t.Fatalf("filter: %v", err)
}
for _, lang := range data.Locales() {
p := message.NewPrinter(language.MustParse(lang))
ldml := data.RawLDML(lang)
if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
continue
}
for _, c := range ldml.Numbers.Currencies.Currency {
syms := cldr.MakeSlice(&c.Symbol)
syms.SelectDraft(dr)
for _, sym := range c.Symbol {
cur, err := ParseISO(c.Type)
if err != nil {
continue
}
formatter := Symbol
switch sym.Alt {
case "":
case "narrow":
formatter = NarrowSymbol
default:
continue
}
want := sym.Data()
if got := p.Sprint(formatter(cur)); got != want {
t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
}
}
}
}
for _, reg := range data.Supplemental().CurrencyData.Region {
i := 0
for ; regionData[i].Region().String() != reg.Iso3166; i++ {
}
it := Query(Historical, NonTender, Region(language.MustParseRegion(reg.Iso3166)))
for _, cur := range reg.Currency {
from, _ := time.Parse("2006-01-02", cur.From)
to, _ := time.Parse("2006-01-02", cur.To)
it.Next()
for j, r := range []QueryIter{&iter{regionInfo: ®ionData[i]}, it} {
if got, _ := r.From(); from != got {
t.Errorf("%d:%s:%s:from: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, from)
}
if got, _ := r.To(); to != got {
t.Errorf("%d:%s:%s:to: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, to)
}
}
i++
}
}
}
func main() {
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer func() {
buf := &bytes.Buffer{}
if _, err = w.WriteGo(buf, "language"); err != nil {
log.Fatalf("Error formatting file index.go: %v", err)
}
// Since we're generating a table for our own package we need to rewrite
// doing the equivalent of go fmt -r 'language.b -> b'. Using
// bytes.Replace will do.
out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
log.Fatalf("Could not create file index.go: %v", err)
}
}()
m := map[language.Tag]bool{}
for _, lang := range data.Locales() {
// We include all locales unconditionally to be consistent with en_US.
// We want en_US, even though it has no data associated with it.
// TODO: put any of the languages for which no data exists at the end
// of the index. This allows all components based on ICU to use that
// as the cutoff point.
// if x := data.RawLDML(lang); false ||
// x.LocaleDisplayNames != nil ||
// x.Characters != nil ||
// x.Delimiters != nil ||
// x.Measurement != nil ||
// x.Dates != nil ||
// x.Numbers != nil ||
// x.Units != nil ||
// x.ListPatterns != nil ||
// x.Collations != nil ||
// x.Segmentations != nil ||
// x.Rbnf != nil ||
// x.Annotations != nil ||
// x.Metadata != nil {
// TODO: support POSIX natively, albeit non-standard.
tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
m[tag] = true
// }
}
// Include locales for plural rules, which uses a different structure.
for _, plurals := range data.Supplemental().Plurals {
for _, rules := range plurals.PluralRules {
for _, lang := range strings.Split(rules.Locales, " ") {
m[language.Make(lang)] = true
}
}
}
var core, special []language.Tag
for t := range m {
if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
log.Fatalf("Unexpected extension %v in %v", x, t)
}
if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
core = append(core, t)
} else {
special = append(special, t)
}
}
w.WriteComment(`
NumCompactTags is the number of common tags. The maximum tag is
NumCompactTags-1.`)
w.WriteConst("NumCompactTags", len(core)+len(special))
sort.Sort(byAlpha(special))
w.WriteVar("specialTags", special)
// TODO: order by frequency?
sort.Sort(byAlpha(core))
// Size computations are just an estimate.
w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size())
w.Size += len(core) * 6 // size of uint32 and uint16
fmt.Fprintln(w)
fmt.Fprintln(w, "var coreTags = map[uint32]uint16{")
fmt.Fprintln(w, "0x0: 0, // und")
i := len(special) + 1 // Und and special tags already written.
for _, t := range core {
if t == language.Und {
continue
}
fmt.Fprint(w.Hash, t, i)
b, s, r := t.Raw()
fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n",
getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number
getIndex(s, 2),
getIndex(r, 3),
i, t)
i++
}
fmt.Fprintln(w, "}")
}
func main() {
gen.Init()
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
data, err := d.DecodeZip(r)
if err != nil {
log.Fatalf("DecodeZip: %v", err)
}
w := gen.NewCodeWriter()
defer func() {
buf := &bytes.Buffer{}
if _, err = w.WriteGo(buf, "language"); err != nil {
log.Fatalf("Error formatting file index.go: %v", err)
}
// Since we're generating a table for our own package we need to rewrite
// doing the equivalent of go fmt -r 'language.b -> b'. Using
// bytes.Replace will do.
out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
log.Fatalf("Could not create file index.go: %v", err)
}
}()
m := map[language.Tag]bool{}
for _, lang := range data.Locales() {
if x := data.RawLDML(lang); false ||
x.LocaleDisplayNames != nil ||
x.Characters != nil ||
x.Delimiters != nil ||
x.Measurement != nil ||
x.Dates != nil ||
x.Numbers != nil ||
x.Units != nil ||
x.ListPatterns != nil ||
x.Collations != nil ||
x.Segmentations != nil ||
x.Rbnf != nil ||
x.Annotations != nil ||
x.Metadata != nil {
// TODO: support POSIX natively, albeit non-standard.
tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
m[tag] = true
}
}
var core, special []language.Tag
for t := range m {
if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
log.Fatalf("Unexpected extension %v in %v", x, t)
}
if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
core = append(core, t)
} else {
special = append(special, t)
}
}
w.WriteComment(`
NumCompactTags is the number of common tags. The maximum tag is
NumCompactTags-1.`)
w.WriteConst("NumCompactTags", len(core)+len(special))
sort.Sort(byAlpha(special))
w.WriteVar("specialTags", special)
type coreKey struct {
base language.Base
script language.Script
region language.Region
}
w.WriteType(coreKey{})
// TODO: order by frequency?
sort.Sort(byAlpha(core))
// Size computations are just an estimate.
w.Size += int(reflect.TypeOf(map[coreKey]uint16{}).Size())
w.Size += len(core) * int(reflect.TypeOf(coreKey{}).Size()+2) // 2 is for uint16
fmt.Fprintln(w, "var coreTags = map[coreKey]uint16{")
fmt.Fprintln(w, "coreKey{}: 0, // und")
i := len(special) + 1 // Und and special tags already written.
for _, t := range core {
if t == language.Und {
continue
}
fmt.Fprint(w.Hash, t, i)
b, s, r := t.Raw()
key := fmt.Sprintf("%#v", coreKey{b, s, r})
key = strings.Replace(key[len("main."):], "language.", "", -1)
fmt.Fprintf(w, "%s: %d, // %s\n", key, i, t)
i++
}
fmt.Fprintln(w, "}")
}