func TestName(t *testing.T) {
testtext.SkipIfNotLong(t)
wants := make([]string, 1+unicode.MaxRune)
ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
r, s := p.Rune(0), p.String(ucd.Name)
if s == "" {
return
}
if s[0] == '<' {
const first = ", First>"
if i := strings.Index(s, first); i >= 0 {
s = s[:i] + ">"
}
}
wants[r] = s
})
nErrors := 0
for r, want := range wants {
got := Name(rune(r))
if got != want {
t.Errorf("r=%#08x: got %q, want %q", r, got, want)
nErrors++
if nErrors == 100 {
t.Fatal("too many errors")
}
}
}
}
// TestBidiCore performs the tests in BidiTest.txt.
// See http://www.unicode.org/Public/UCD/latest/ucd/BidiTest.txt.
func TestBidiCore(t *testing.T) {
testtext.SkipIfNotLong(t)
r := gen.OpenUCDFile("BidiTest.txt")
defer r.Close()
var wantLevels, wantOrder []string
p := ucd.New(r, ucd.Part(func(p *ucd.Parser) {
s := strings.Split(p.String(0), ":")
switch s[0] {
case "Levels":
wantLevels = strings.Fields(s[1])
case "Reorder":
wantOrder = strings.Fields(s[1])
default:
log.Fatalf("Unknown part %q.", s[0])
}
}))
for p.Next() {
types := []class{}
for _, s := range p.Strings(0) {
types = append(types, bidiClass[s])
}
// We ignore the bracketing part of the algorithm.
pairTypes := make([]bracketType, len(types))
pairValues := make([]rune, len(types))
for i := uint(0); i < 3; i++ {
if p.Uint(1)&(1<<i) == 0 {
continue
}
lev := level(int(i) - 1)
par := newParagraph(types, pairTypes, pairValues, lev)
if *testLevels {
levels := par.resultLevels
for i, s := range wantLevels {
if s == "x" {
continue
}
l, _ := strconv.ParseUint(s, 10, 8)
if level(l)&1 != levels[i]&1 {
t.Errorf("%s:%d:levels: got %v; want %v", p.String(0), lev, levels, wantLevels)
break
}
}
}
order := par.getReordering([]int{len(types)})
gotOrder := filterOrder(types, order)
if got, want := fmt.Sprint(gotOrder), fmt.Sprint(wantOrder); got != want {
t.Errorf("%s:%d:order: got %v; want %v\noriginal %v", p.String(0), lev, got, want, order)
}
}
}
if err := p.Err(); err != nil {
log.Fatal(err)
}
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
runes := map[rune]Kind{}
getWidthData(func(r rune, tag elem, _ rune) {
runes[r] = tag.kind()
})
for r := rune(0); r < 0x10FFFF; r++ {
if loSurrogate <= r && r <= hiSurrogate {
continue
}
p := LookupRune(r)
if got, want := p.Kind(), runes[r]; got != want {
t.Errorf("Kind of %U was %s; want %s.", r, got, want)
}
want, mapped := foldRune(r)
if got := p.Folded(); (got == 0) == mapped || got != 0 && got != want {
t.Errorf("Folded(%U) = %U; want %U", r, got, want)
}
want, mapped = widenRune(r)
if got := p.Wide(); (got == 0) == mapped || got != 0 && got != want {
t.Errorf("Wide(%U) = %U; want %U", r, got, want)
}
want, mapped = narrowRune(r)
if got := p.Narrow(); (got == 0) == mapped || got != 0 && got != want {
t.Errorf("Narrow(%U) = %U; want %U", r, got, want)
}
}
}
func TestNumberSystems(t *testing.T) {
testtext.SkipIfNotLong(t)
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental")
d.SetSectionFilter("numberingSystem")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
n := systemMap[ns.Id]
if int(n) >= len(numSysData) {
continue
}
info := InfoFromLangID(0, ns.Id)
val := '0'
for _, rWant := range ns.Digits {
if rGot := info.Digit(val); rGot != rWant {
t.Errorf("%s:%d: got %U; want %U", ns.Id, val, rGot, rWant)
}
val++
}
}
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
dr, err := cldr.ParseDraft(*draft)
if err != nil {
t.Fatalf("filter: %v", err)
}
for _, lang := range data.Locales() {
p := message.NewPrinter(language.MustParse(lang))
ldml := data.RawLDML(lang)
if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
continue
}
for _, c := range ldml.Numbers.Currencies.Currency {
syms := cldr.MakeSlice(&c.Symbol)
syms.SelectDraft(dr)
for _, sym := range c.Symbol {
cur, err := ParseISO(c.Type)
if err != nil {
continue
}
formatter := Symbol
switch sym.Alt {
case "":
case "narrow":
formatter = NarrowSymbol
default:
continue
}
want := sym.Data()
if got := p.Sprint(formatter(cur)); got != want {
t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
}
}
}
}
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) {
r1 := p.Rune(0)
want := p.Rune(1)
e, _ := LookupRune(r1)
if got := e.reverseBracket(r1); got != want {
t.Errorf("Reverse(%U) = %U; want %U", r1, got, want)
}
})
done := map[rune]bool{}
test := func(name string, r rune, want string) {
str := string(r)
e, _ := LookupString(str)
if got := labels[e.Class()]; got != want {
t.Errorf("%s:%U: got %s; want %s", name, r, got, want)
}
if e2, sz := LookupRune(r); e != e2 || sz != len(str) {
t.Errorf("LookupRune(%U) = %v, %d; want %v, %d", r, e2, e, sz, len(str))
}
if e2, sz := Lookup([]byte(str)); e != e2 || sz != len(str) {
t.Errorf("Lookup(%U) = %v, %d; want %v, %d", r, e2, e, sz, len(str))
}
done[r] = true
}
// Insert the derived BiDi properties.
ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) {
r := p.Rune(0)
test("derived", r, p.String(1))
})
visitDefaults(func(r rune, c Class) {
if !done[r] {
test("default", r, labels[c])
}
})
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
trie := newBidiTrie(0)
ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) {
r1 := p.Rune(0)
want := p.Rune(1)
e, _ := trie.lookupString(string(r1))
if got := entry(e).reverseBracket(r1); got != want {
t.Errorf("Reverse(%U) = %U; want %U", r1, got, want)
}
})
done := map[rune]bool{}
test := func(name string, r rune, want string) {
e, _ := trie.lookupString(string(r))
if got := labels[entry(e).class(r)]; got != want {
t.Errorf("%s:%U: got %s; want %s", name, r, got, want)
}
done[r] = true
}
// Insert the derived BiDi properties.
ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) {
r := p.Rune(0)
test("derived", r, p.String(1))
})
visitDefaults(func(r rune, c class) {
if !done[r] {
test("default", r, labels[c])
}
})
}
// TestBidiCharacters performs the tests in BidiCharacterTest.txt.
// See http://www.unicode.org/Public/UCD/latest/ucd/BidiCharacterTest.txt
func TestBidiCharacters(t *testing.T) {
testtext.SkipIfNotLong(t)
ucd.Parse(gen.OpenUCDFile("BidiCharacterTest.txt"), func(p *ucd.Parser) {
var (
types []class
pairTypes []bracketType
pairValues []rune
parLevel level
wantLevel = level(p.Int(2))
wantLevels = p.Strings(3)
wantVisualOrder = p.Strings(4)
)
switch l := p.Int(1); l {
case 0, 1:
parLevel = level(l)
case 2:
parLevel = implicitLevel
default:
// Spec says to ignore unknown parts.
}
trie := newBidiTrie(0)
runes := p.Runes(0)
for _, r := range runes {
// Assign the bracket type.
if d := norm.NFKD.PropertiesString(string(r)).Decomposition(); d != nil {
r = []rune(string(d))[0]
}
e, _ := trie.lookupString(string(r))
entry := entry(e)
// Assign the class for this rune.
types = append(types, entry.class(r))
switch {
case !entry.isBracket():
pairTypes = append(pairTypes, bpNone)
pairValues = append(pairValues, 0)
case entry.isOpen():
pairTypes = append(pairTypes, bpOpen)
pairValues = append(pairValues, r)
default:
pairTypes = append(pairTypes, bpClose)
pairValues = append(pairValues, entry.reverseBracket(r))
}
}
par := newParagraph(types, pairTypes, pairValues, parLevel)
// Test results:
if got := par.embeddingLevel; got != wantLevel {
t.Errorf("%v:level: got %d; want %d", string(runes), got, wantLevel)
}
if *testLevels {
gotLevels := getLevelStrings(types, par.resultLevels)
if got, want := fmt.Sprint(gotLevels), fmt.Sprint(wantLevels); got != want {
t.Errorf("%04X %q:%d: got %v; want %v\nval: %x\npair: %v", runes, string(runes), parLevel, got, want, pairValues, pairTypes)
}
}
order := par.getReordering([]int{len(types)})
order = filterOrder(types, order)
if got, want := fmt.Sprint(order), fmt.Sprint(wantVisualOrder); got != want {
t.Errorf("%04X %q:%d: got %v; want %v\ngot order: %s", runes, string(runes), parLevel, got, want, reorder(runes, order))
}
})
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
lookup := func(r rune) info {
v, _ := trie.lookupString(string(r))
return info(v)
}
ucd.Parse(gen.OpenUnicodeFile("idna", "", "IdnaMappingTable.txt"), func(p *ucd.Parser) {
r := p.Rune(0)
x := lookup(r)
if got, want := x.category(), catFromEntry(p); got != want {
t.Errorf("%U:category: got %x; want %x", r, got, want)
}
mapped := false
switch p.String(1) {
case "mapped", "disallowed_STD3_mapped", "deviation":
mapped = true
}
if x.isMapped() != mapped {
t.Errorf("%U:isMapped: got %v; want %v", r, x.isMapped(), mapped)
}
if !mapped {
return
}
want := string(p.Runes(2))
got := string(x.appendMapping(nil, string(r)))
if got != want {
t.Errorf("%U:mapping: got %+q; want %+q", r, got, want)
}
if x.isMapped() {
return
}
wantMark := unicode.In(r, unicode.Mark)
gotMark := x.isModifier()
if gotMark != wantMark {
t.Errorf("IsMark(%U) = %v; want %v", r, gotMark, wantMark)
}
})
ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
r := p.Rune(0)
x := lookup(r)
got := x.isViramaModifier()
const cccVirama = 9
want := p.Int(ucd.CanonicalCombiningClass) == cccVirama
if got != want {
t.Errorf("IsVirama(%U) = %v; want %v", r, got, want)
}
})
ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) {
r := p.Rune(0)
x := lookup(r)
if x.isMapped() {
return
}
got := x.joinType()
want := joinType[p.String(1)]
if got != want {
t.Errorf("JoinType(%U) = %x; want %x", r, got, want)
}
})
}
func TestConformance(t *testing.T) {
testtext.SkipIfNotLong(t)
r := gen.OpenUnicodeFile("idna", "", "IdnaTest.txt")
defer r.Close()
section := "main"
started := false
p := ucd.New(r, ucd.CommentHandler(func(s string) {
if started {
section = strings.ToLower(strings.Split(s, " ")[0])
}
}))
transitional := New(Transitional(true), VerifyDNSLength(true))
nonTransitional := New(VerifyDNSLength(true))
for p.Next() {
started = true
// What to test
profiles := []*Profile{}
switch p.String(0) {
case "T":
profiles = append(profiles, transitional)
case "N":
profiles = append(profiles, nonTransitional)
case "B":
profiles = append(profiles, transitional)
profiles = append(profiles, nonTransitional)
}
src := unescape(p.String(1))
wantToUnicode := unescape(p.String(2))
if wantToUnicode == "" {
wantToUnicode = src
}
wantToASCII := unescape(p.String(3))
if wantToASCII == "" {
wantToASCII = wantToUnicode
}
wantErrToUnicode := ""
if strings.HasPrefix(wantToUnicode, "[") {
wantErrToUnicode = wantToUnicode
wantToUnicode = ""
}
wantErrToASCII := ""
if strings.HasPrefix(wantToASCII, "[") {
wantErrToASCII = wantToASCII
wantToASCII = ""
}
// TODO: also do IDNA tests.
// invalidInIDNA2008 := p.String(4) == "NV8"
for _, p := range profiles {
name := fmt.Sprintf("%s:%s", section, p)
doTest(t, p.ToUnicode, name+":ToUnicode", src, wantToUnicode, wantErrToUnicode)
doTest(t, p.ToASCII, name+":ToASCII", src, wantToASCII, wantErrToASCII)
}
}
}
func TestTables(t *testing.T) {
testtext.SkipIfNotLong(t)
// Read the CLDR zip file.
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("supplemental", "main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
dr, err := cldr.ParseDraft(*draft)
if err != nil {
t.Fatalf("filter: %v", err)
}
for _, lang := range data.Locales() {
p := message.NewPrinter(language.MustParse(lang))
ldml := data.RawLDML(lang)
if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
continue
}
for _, c := range ldml.Numbers.Currencies.Currency {
syms := cldr.MakeSlice(&c.Symbol)
syms.SelectDraft(dr)
for _, sym := range c.Symbol {
cur, err := ParseISO(c.Type)
if err != nil {
continue
}
formatter := Symbol
switch sym.Alt {
case "":
case "narrow":
formatter = NarrowSymbol
default:
continue
}
want := sym.Data()
if got := p.Sprint(formatter(cur)); got != want {
t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
}
}
}
}
for _, reg := range data.Supplemental().CurrencyData.Region {
i := 0
for ; regionData[i].Region().String() != reg.Iso3166; i++ {
}
it := Query(Historical, NonTender, Region(language.MustParseRegion(reg.Iso3166)))
for _, cur := range reg.Currency {
from, _ := time.Parse("2006-01-02", cur.From)
to, _ := time.Parse("2006-01-02", cur.To)
it.Next()
for j, r := range []QueryIter{&iter{regionInfo: ®ionData[i]}, it} {
if got, _ := r.From(); from != got {
t.Errorf("%d:%s:%s:from: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, from)
}
if got, _ := r.To(); to != got {
t.Errorf("%d:%s:%s:to: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, to)
}
}
i++
}
}
}
func TestConformance(t *testing.T) {
testtext.SkipIfNotLong(t)
r := gen.OpenUnicodeFile("idna", "", "IdnaTest.txt")
defer r.Close()
section := "main"
started := false
p := ucd.New(r, ucd.CommentHandler(func(s string) {
if started {
section = strings.ToLower(strings.Split(s, " ")[0])
}
}))
for p.Next() {
started = true
// What to test
profiles := []*Profile{}
switch p.String(0) {
case "T":
profiles = append(profiles, Transitional)
case "N":
profiles = append(profiles, NonTransitional)
case "B":
profiles = append(profiles, Transitional)
profiles = append(profiles, NonTransitional)
}
src := unescape(p.String(1))
if incorrectTests[src] {
continue
}
wantToUnicode := unescape(p.String(2))
if wantToUnicode == "" {
wantToUnicode = src
}
wantToASCII := unescape(p.String(3))
if wantToASCII == "" {
wantToASCII = wantToUnicode
}
test := "err:"
if strings.HasPrefix(wantToUnicode, "[") {
test += strings.Replace(strings.Trim(wantToUnicode, "[]"), " ", "", -1)
}
if strings.HasPrefix(wantToASCII, "[") {
test += strings.Replace(strings.Trim(wantToASCII, "[]"), " ", "", -1)
}
if test == "err:" {
test = "ok"
}
// TODO: also do IDNA tests.
// invalidInIDNA2008 := p.String(4) == "NV8"
for _, p := range profiles {
testtext.Run(t, fmt.Sprintf("%s:%s/%s/%+q", section, test, p, src), func(t *testing.T) {
got, err := p.ToUnicode(src)
wantErr := strings.HasPrefix(wantToUnicode, "[")
gotErr := err != nil
if wantErr {
if gotErr != wantErr {
t.Errorf(`ToUnicode:err got %v; want %v (%s)`,
gotErr, wantErr, wantToUnicode)
}
} else if got != wantToUnicode || gotErr != wantErr {
t.Errorf(`ToUnicode: got %+q, %v (%v); want %+q, %v`,
got, gotErr, err, wantToUnicode, wantErr)
}
got, err = p.ToASCII(src)
wantErr = strings.HasPrefix(wantToASCII, "[")
gotErr = err != nil
if wantErr {
if gotErr != wantErr {
t.Errorf(`ToASCII:err got %v; want %v (%s)`,
gotErr, wantErr, wantToASCII)
}
} else if got != wantToASCII || gotErr != wantErr {
t.Errorf(`ToASCII: got %+q, %v (%v); want %+q, %v`,
got, gotErr, err, wantToASCII, wantErr)
}
})
}
}
}
func TestSymbols(t *testing.T) {
testtext.SkipIfNotLong(t)
draft, err := cldr.ParseDraft(*draft)
if err != nil {
log.Fatalf("invalid draft level: %v", err)
}
r := gen.OpenCLDRCoreZip()
defer r.Close()
d := &cldr.Decoder{}
d.SetDirFilter("main")
d.SetSectionFilter("numbers")
data, err := d.DecodeZip(r)
if err != nil {
t.Fatalf("DecodeZip: %v", err)
}
for _, lang := range data.Locales() {
ldml := data.RawLDML(lang)
if ldml.Numbers == nil {
continue
}
langIndex, ok := language.CompactIndex(language.MustParse(lang))
if !ok {
t.Fatalf("No compact index for language %s", lang)
}
syms := cldr.MakeSlice(&ldml.Numbers.Symbols)
syms.SelectDraft(draft)
for _, sym := range ldml.Numbers.Symbols {
if sym.NumberSystem == "" {
continue
}
testCases := []struct {
name string
st SymbolType
x interface{}
}{
{"Decimal", SymDecimal, sym.Decimal},
{"Group", SymGroup, sym.Group},
{"List", SymList, sym.List},
{"PercentSign", SymPercentSign, sym.PercentSign},
{"PlusSign", SymPlusSign, sym.PlusSign},
{"MinusSign", SymMinusSign, sym.MinusSign},
{"Exponential", SymExponential, sym.Exponential},
{"SuperscriptingExponent", SymSuperscriptingExponent, sym.SuperscriptingExponent},
{"PerMille", SymPerMille, sym.PerMille},
{"Infinity", SymInfinity, sym.Infinity},
{"NaN", SymNan, sym.Nan},
{"TimeSeparator", SymTimeSeparator, sym.TimeSeparator},
}
info := InfoFromLangID(langIndex, sym.NumberSystem)
for _, tc := range testCases {
// Extract the wanted value.
v := reflect.ValueOf(tc.x)
if v.Len() == 0 {
return
}
if v.Len() > 1 {
t.Fatalf("Multiple values of %q within single symbol not supported.", tc.name)
}
want := v.Index(0).MethodByName("Data").Call(nil)[0].String()
got := info.Symbol(tc.st)
if got != want {
t.Errorf("%s:%s:%s: got %q; want %q", lang, sym.NumberSystem, tc.name, got, want)
}
}
}
}
}
func skipShort(t *testing.T) {
testtext.SkipIfNotLong(t)
once.Do(func() { loadTestData(t) })
}