Exemple #1
0
func TestName(t *testing.T) {
	testtext.SkipIfNotLong(t)

	wants := make([]string, 1+unicode.MaxRune)
	ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
		r, s := p.Rune(0), p.String(ucd.Name)
		if s == "" {
			return
		}
		if s[0] == '<' {
			const first = ", First>"
			if i := strings.Index(s, first); i >= 0 {
				s = s[:i] + ">"
			}
		}
		wants[r] = s
	})

	nErrors := 0
	for r, want := range wants {
		got := Name(rune(r))
		if got != want {
			t.Errorf("r=%#08x: got %q, want %q", r, got, want)
			nErrors++
			if nErrors == 100 {
				t.Fatal("too many errors")
			}
		}
	}
}
// TestBidiCore performs the tests in BidiTest.txt.
// See http://www.unicode.org/Public/UCD/latest/ucd/BidiTest.txt.
func TestBidiCore(t *testing.T) {
	testtext.SkipIfNotLong(t)

	r := gen.OpenUCDFile("BidiTest.txt")
	defer r.Close()

	var wantLevels, wantOrder []string
	p := ucd.New(r, ucd.Part(func(p *ucd.Parser) {
		s := strings.Split(p.String(0), ":")
		switch s[0] {
		case "Levels":
			wantLevels = strings.Fields(s[1])
		case "Reorder":
			wantOrder = strings.Fields(s[1])
		default:
			log.Fatalf("Unknown part %q.", s[0])
		}
	}))

	for p.Next() {
		types := []class{}
		for _, s := range p.Strings(0) {
			types = append(types, bidiClass[s])
		}
		// We ignore the bracketing part of the algorithm.
		pairTypes := make([]bracketType, len(types))
		pairValues := make([]rune, len(types))

		for i := uint(0); i < 3; i++ {
			if p.Uint(1)&(1<<i) == 0 {
				continue
			}
			lev := level(int(i) - 1)
			par := newParagraph(types, pairTypes, pairValues, lev)

			if *testLevels {
				levels := par.resultLevels
				for i, s := range wantLevels {
					if s == "x" {
						continue
					}
					l, _ := strconv.ParseUint(s, 10, 8)
					if level(l)&1 != levels[i]&1 {
						t.Errorf("%s:%d:levels: got %v; want %v", p.String(0), lev, levels, wantLevels)
						break
					}
				}
			}

			order := par.getReordering([]int{len(types)})
			gotOrder := filterOrder(types, order)
			if got, want := fmt.Sprint(gotOrder), fmt.Sprint(wantOrder); got != want {
				t.Errorf("%s:%d:order: got %v; want %v\noriginal %v", p.String(0), lev, got, want, order)
			}
		}
	}
	if err := p.Err(); err != nil {
		log.Fatal(err)
	}
}
Exemple #3
0
func TestTables(t *testing.T) {
	testtext.SkipIfNotLong(t)

	runes := map[rune]Kind{}
	getWidthData(func(r rune, tag elem, _ rune) {
		runes[r] = tag.kind()
	})
	for r := rune(0); r < 0x10FFFF; r++ {
		if loSurrogate <= r && r <= hiSurrogate {
			continue
		}
		p := LookupRune(r)
		if got, want := p.Kind(), runes[r]; got != want {
			t.Errorf("Kind of %U was %s; want %s.", r, got, want)
		}
		want, mapped := foldRune(r)
		if got := p.Folded(); (got == 0) == mapped || got != 0 && got != want {
			t.Errorf("Folded(%U) = %U; want %U", r, got, want)
		}
		want, mapped = widenRune(r)
		if got := p.Wide(); (got == 0) == mapped || got != 0 && got != want {
			t.Errorf("Wide(%U) = %U; want %U", r, got, want)
		}
		want, mapped = narrowRune(r)
		if got := p.Narrow(); (got == 0) == mapped || got != 0 && got != want {
			t.Errorf("Narrow(%U) = %U; want %U", r, got, want)
		}
	}
}
Exemple #4
0
func TestNumberSystems(t *testing.T) {
	testtext.SkipIfNotLong(t)

	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental")
	d.SetSectionFilter("numberingSystem")
	data, err := d.DecodeZip(r)
	if err != nil {
		t.Fatalf("DecodeZip: %v", err)
	}

	for _, ns := range data.Supplemental().NumberingSystems.NumberingSystem {
		n := systemMap[ns.Id]
		if int(n) >= len(numSysData) {
			continue
		}
		info := InfoFromLangID(0, ns.Id)
		val := '0'
		for _, rWant := range ns.Digits {
			if rGot := info.Digit(val); rGot != rWant {
				t.Errorf("%s:%d: got %U; want %U", ns.Id, val, rGot, rWant)
			}
			val++
		}
	}
}
func TestTables(t *testing.T) {
	testtext.SkipIfNotLong(t)

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental", "main")
	d.SetSectionFilter("numbers")
	data, err := d.DecodeZip(r)
	if err != nil {
		t.Fatalf("DecodeZip: %v", err)
	}

	dr, err := cldr.ParseDraft(*draft)
	if err != nil {
		t.Fatalf("filter: %v", err)
	}

	for _, lang := range data.Locales() {
		p := message.NewPrinter(language.MustParse(lang))

		ldml := data.RawLDML(lang)
		if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
			continue
		}
		for _, c := range ldml.Numbers.Currencies.Currency {
			syms := cldr.MakeSlice(&c.Symbol)
			syms.SelectDraft(dr)

			for _, sym := range c.Symbol {
				cur, err := ParseISO(c.Type)
				if err != nil {
					continue
				}
				formatter := Symbol
				switch sym.Alt {
				case "":
				case "narrow":
					formatter = NarrowSymbol
				default:
					continue
				}
				want := sym.Data()
				if got := p.Sprint(formatter(cur)); got != want {
					t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
				}
			}
		}
	}
}
Exemple #6
0
func TestTables(t *testing.T) {
	testtext.SkipIfNotLong(t)

	ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) {
		r1 := p.Rune(0)
		want := p.Rune(1)

		e, _ := LookupRune(r1)
		if got := e.reverseBracket(r1); got != want {
			t.Errorf("Reverse(%U) = %U; want %U", r1, got, want)
		}
	})

	done := map[rune]bool{}
	test := func(name string, r rune, want string) {
		str := string(r)
		e, _ := LookupString(str)
		if got := labels[e.Class()]; got != want {
			t.Errorf("%s:%U: got %s; want %s", name, r, got, want)
		}
		if e2, sz := LookupRune(r); e != e2 || sz != len(str) {
			t.Errorf("LookupRune(%U) = %v, %d; want %v, %d", r, e2, e, sz, len(str))
		}
		if e2, sz := Lookup([]byte(str)); e != e2 || sz != len(str) {
			t.Errorf("Lookup(%U) = %v, %d; want %v, %d", r, e2, e, sz, len(str))
		}
		done[r] = true
	}

	// Insert the derived BiDi properties.
	ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) {
		r := p.Rune(0)
		test("derived", r, p.String(1))
	})
	visitDefaults(func(r rune, c Class) {
		if !done[r] {
			test("default", r, labels[c])
		}
	})

}
func TestTables(t *testing.T) {
	testtext.SkipIfNotLong(t)

	trie := newBidiTrie(0)

	ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) {
		r1 := p.Rune(0)
		want := p.Rune(1)

		e, _ := trie.lookupString(string(r1))
		if got := entry(e).reverseBracket(r1); got != want {
			t.Errorf("Reverse(%U) = %U; want %U", r1, got, want)
		}
	})

	done := map[rune]bool{}
	test := func(name string, r rune, want string) {
		e, _ := trie.lookupString(string(r))
		if got := labels[entry(e).class(r)]; got != want {
			t.Errorf("%s:%U: got %s; want %s", name, r, got, want)
		}
		done[r] = true
	}

	// Insert the derived BiDi properties.
	ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) {
		r := p.Rune(0)
		test("derived", r, p.String(1))
	})
	visitDefaults(func(r rune, c class) {
		if !done[r] {
			test("default", r, labels[c])
		}
	})

}
// TestBidiCharacters performs the tests in BidiCharacterTest.txt.
// See http://www.unicode.org/Public/UCD/latest/ucd/BidiCharacterTest.txt
func TestBidiCharacters(t *testing.T) {
	testtext.SkipIfNotLong(t)

	ucd.Parse(gen.OpenUCDFile("BidiCharacterTest.txt"), func(p *ucd.Parser) {
		var (
			types      []class
			pairTypes  []bracketType
			pairValues []rune
			parLevel   level

			wantLevel       = level(p.Int(2))
			wantLevels      = p.Strings(3)
			wantVisualOrder = p.Strings(4)
		)

		switch l := p.Int(1); l {
		case 0, 1:
			parLevel = level(l)
		case 2:
			parLevel = implicitLevel
		default:
			// Spec says to ignore unknown parts.
		}

		trie := newBidiTrie(0)
		runes := p.Runes(0)

		for _, r := range runes {
			// Assign the bracket type.
			if d := norm.NFKD.PropertiesString(string(r)).Decomposition(); d != nil {
				r = []rune(string(d))[0]
			}
			e, _ := trie.lookupString(string(r))
			entry := entry(e)

			// Assign the class for this rune.
			types = append(types, entry.class(r))

			switch {
			case !entry.isBracket():
				pairTypes = append(pairTypes, bpNone)
				pairValues = append(pairValues, 0)
			case entry.isOpen():
				pairTypes = append(pairTypes, bpOpen)
				pairValues = append(pairValues, r)
			default:
				pairTypes = append(pairTypes, bpClose)
				pairValues = append(pairValues, entry.reverseBracket(r))
			}
		}
		par := newParagraph(types, pairTypes, pairValues, parLevel)

		// Test results:
		if got := par.embeddingLevel; got != wantLevel {
			t.Errorf("%v:level: got %d; want %d", string(runes), got, wantLevel)
		}

		if *testLevels {
			gotLevels := getLevelStrings(types, par.resultLevels)
			if got, want := fmt.Sprint(gotLevels), fmt.Sprint(wantLevels); got != want {
				t.Errorf("%04X %q:%d: got %v; want %v\nval: %x\npair: %v", runes, string(runes), parLevel, got, want, pairValues, pairTypes)
			}
		}

		order := par.getReordering([]int{len(types)})
		order = filterOrder(types, order)
		if got, want := fmt.Sprint(order), fmt.Sprint(wantVisualOrder); got != want {
			t.Errorf("%04X %q:%d: got %v; want %v\ngot order: %s", runes, string(runes), parLevel, got, want, reorder(runes, order))
		}
	})
}
Exemple #9
0
func TestTables(t *testing.T) {
	testtext.SkipIfNotLong(t)

	lookup := func(r rune) info {
		v, _ := trie.lookupString(string(r))
		return info(v)
	}

	ucd.Parse(gen.OpenUnicodeFile("idna", "", "IdnaMappingTable.txt"), func(p *ucd.Parser) {
		r := p.Rune(0)
		x := lookup(r)
		if got, want := x.category(), catFromEntry(p); got != want {
			t.Errorf("%U:category: got %x; want %x", r, got, want)
		}

		mapped := false
		switch p.String(1) {
		case "mapped", "disallowed_STD3_mapped", "deviation":
			mapped = true
		}
		if x.isMapped() != mapped {
			t.Errorf("%U:isMapped: got %v; want %v", r, x.isMapped(), mapped)
		}
		if !mapped {
			return
		}
		want := string(p.Runes(2))
		got := string(x.appendMapping(nil, string(r)))
		if got != want {
			t.Errorf("%U:mapping: got %+q; want %+q", r, got, want)
		}

		if x.isMapped() {
			return
		}
		wantMark := unicode.In(r, unicode.Mark)
		gotMark := x.isModifier()
		if gotMark != wantMark {
			t.Errorf("IsMark(%U) = %v; want %v", r, gotMark, wantMark)
		}
	})

	ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
		r := p.Rune(0)
		x := lookup(r)
		got := x.isViramaModifier()

		const cccVirama = 9
		want := p.Int(ucd.CanonicalCombiningClass) == cccVirama
		if got != want {
			t.Errorf("IsVirama(%U) = %v; want %v", r, got, want)
		}
	})

	ucd.Parse(gen.OpenUCDFile("extracted/DerivedJoiningType.txt"), func(p *ucd.Parser) {
		r := p.Rune(0)
		x := lookup(r)
		if x.isMapped() {
			return
		}
		got := x.joinType()
		want := joinType[p.String(1)]
		if got != want {
			t.Errorf("JoinType(%U) = %x; want %x", r, got, want)
		}
	})
}
Exemple #10
0
func TestConformance(t *testing.T) {
	testtext.SkipIfNotLong(t)

	r := gen.OpenUnicodeFile("idna", "", "IdnaTest.txt")
	defer r.Close()

	section := "main"
	started := false
	p := ucd.New(r, ucd.CommentHandler(func(s string) {
		if started {
			section = strings.ToLower(strings.Split(s, " ")[0])
		}
	}))
	transitional := New(Transitional(true), VerifyDNSLength(true))
	nonTransitional := New(VerifyDNSLength(true))
	for p.Next() {
		started = true

		// What to test
		profiles := []*Profile{}
		switch p.String(0) {
		case "T":
			profiles = append(profiles, transitional)
		case "N":
			profiles = append(profiles, nonTransitional)
		case "B":
			profiles = append(profiles, transitional)
			profiles = append(profiles, nonTransitional)
		}

		src := unescape(p.String(1))

		wantToUnicode := unescape(p.String(2))
		if wantToUnicode == "" {
			wantToUnicode = src
		}
		wantToASCII := unescape(p.String(3))
		if wantToASCII == "" {
			wantToASCII = wantToUnicode
		}
		wantErrToUnicode := ""
		if strings.HasPrefix(wantToUnicode, "[") {
			wantErrToUnicode = wantToUnicode
			wantToUnicode = ""
		}
		wantErrToASCII := ""
		if strings.HasPrefix(wantToASCII, "[") {
			wantErrToASCII = wantToASCII
			wantToASCII = ""
		}

		// TODO: also do IDNA tests.
		// invalidInIDNA2008 := p.String(4) == "NV8"

		for _, p := range profiles {
			name := fmt.Sprintf("%s:%s", section, p)
			doTest(t, p.ToUnicode, name+":ToUnicode", src, wantToUnicode, wantErrToUnicode)
			doTest(t, p.ToASCII, name+":ToASCII", src, wantToASCII, wantErrToASCII)
		}
	}
}
Exemple #11
0
func TestTables(t *testing.T) {
	testtext.SkipIfNotLong(t)

	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental", "main")
	d.SetSectionFilter("numbers")
	data, err := d.DecodeZip(r)
	if err != nil {
		t.Fatalf("DecodeZip: %v", err)
	}

	dr, err := cldr.ParseDraft(*draft)
	if err != nil {
		t.Fatalf("filter: %v", err)
	}

	for _, lang := range data.Locales() {
		p := message.NewPrinter(language.MustParse(lang))

		ldml := data.RawLDML(lang)
		if ldml.Numbers == nil || ldml.Numbers.Currencies == nil {
			continue
		}
		for _, c := range ldml.Numbers.Currencies.Currency {
			syms := cldr.MakeSlice(&c.Symbol)
			syms.SelectDraft(dr)

			for _, sym := range c.Symbol {
				cur, err := ParseISO(c.Type)
				if err != nil {
					continue
				}
				formatter := Symbol
				switch sym.Alt {
				case "":
				case "narrow":
					formatter = NarrowSymbol
				default:
					continue
				}
				want := sym.Data()
				if got := p.Sprint(formatter(cur)); got != want {
					t.Errorf("%s:%sSymbol(%s) = %s; want %s", lang, strings.Title(sym.Alt), c.Type, got, want)
				}
			}
		}
	}

	for _, reg := range data.Supplemental().CurrencyData.Region {
		i := 0
		for ; regionData[i].Region().String() != reg.Iso3166; i++ {
		}
		it := Query(Historical, NonTender, Region(language.MustParseRegion(reg.Iso3166)))
		for _, cur := range reg.Currency {
			from, _ := time.Parse("2006-01-02", cur.From)
			to, _ := time.Parse("2006-01-02", cur.To)

			it.Next()
			for j, r := range []QueryIter{&iter{regionInfo: &regionData[i]}, it} {
				if got, _ := r.From(); from != got {
					t.Errorf("%d:%s:%s:from: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, from)
				}
				if got, _ := r.To(); to != got {
					t.Errorf("%d:%s:%s:to: got %v; want %v", j, reg.Iso3166, cur.Iso4217, got, to)
				}
			}
			i++
		}
	}
}
Exemple #12
0
func TestConformance(t *testing.T) {
	testtext.SkipIfNotLong(t)

	r := gen.OpenUnicodeFile("idna", "", "IdnaTest.txt")
	defer r.Close()

	section := "main"
	started := false
	p := ucd.New(r, ucd.CommentHandler(func(s string) {
		if started {
			section = strings.ToLower(strings.Split(s, " ")[0])
		}
	}))
	for p.Next() {
		started = true

		// What to test
		profiles := []*Profile{}
		switch p.String(0) {
		case "T":
			profiles = append(profiles, Transitional)
		case "N":
			profiles = append(profiles, NonTransitional)
		case "B":
			profiles = append(profiles, Transitional)
			profiles = append(profiles, NonTransitional)
		}

		src := unescape(p.String(1))
		if incorrectTests[src] {
			continue
		}

		wantToUnicode := unescape(p.String(2))
		if wantToUnicode == "" {
			wantToUnicode = src
		}
		wantToASCII := unescape(p.String(3))
		if wantToASCII == "" {
			wantToASCII = wantToUnicode
		}
		test := "err:"
		if strings.HasPrefix(wantToUnicode, "[") {
			test += strings.Replace(strings.Trim(wantToUnicode, "[]"), " ", "", -1)
		}
		if strings.HasPrefix(wantToASCII, "[") {
			test += strings.Replace(strings.Trim(wantToASCII, "[]"), " ", "", -1)
		}
		if test == "err:" {
			test = "ok"
		}

		// TODO: also do IDNA tests.
		// invalidInIDNA2008 := p.String(4) == "NV8"

		for _, p := range profiles {
			testtext.Run(t, fmt.Sprintf("%s:%s/%s/%+q", section, test, p, src), func(t *testing.T) {
				got, err := p.ToUnicode(src)
				wantErr := strings.HasPrefix(wantToUnicode, "[")
				gotErr := err != nil
				if wantErr {
					if gotErr != wantErr {
						t.Errorf(`ToUnicode:err got %v; want %v (%s)`,
							gotErr, wantErr, wantToUnicode)
					}
				} else if got != wantToUnicode || gotErr != wantErr {
					t.Errorf(`ToUnicode: got %+q, %v (%v); want %+q, %v`,
						got, gotErr, err, wantToUnicode, wantErr)
				}

				got, err = p.ToASCII(src)
				wantErr = strings.HasPrefix(wantToASCII, "[")
				gotErr = err != nil
				if wantErr {
					if gotErr != wantErr {
						t.Errorf(`ToASCII:err got %v; want %v (%s)`,
							gotErr, wantErr, wantToASCII)
					}
				} else if got != wantToASCII || gotErr != wantErr {
					t.Errorf(`ToASCII: got %+q, %v (%v); want %+q, %v`,
						got, gotErr, err, wantToASCII, wantErr)
				}
			})
		}
	}
}
Exemple #13
0
func TestSymbols(t *testing.T) {
	testtext.SkipIfNotLong(t)

	draft, err := cldr.ParseDraft(*draft)
	if err != nil {
		log.Fatalf("invalid draft level: %v", err)
	}

	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("main")
	d.SetSectionFilter("numbers")
	data, err := d.DecodeZip(r)
	if err != nil {
		t.Fatalf("DecodeZip: %v", err)
	}

	for _, lang := range data.Locales() {
		ldml := data.RawLDML(lang)
		if ldml.Numbers == nil {
			continue
		}
		langIndex, ok := language.CompactIndex(language.MustParse(lang))
		if !ok {
			t.Fatalf("No compact index for language %s", lang)
		}

		syms := cldr.MakeSlice(&ldml.Numbers.Symbols)
		syms.SelectDraft(draft)

		for _, sym := range ldml.Numbers.Symbols {
			if sym.NumberSystem == "" {
				continue
			}
			testCases := []struct {
				name string
				st   SymbolType
				x    interface{}
			}{
				{"Decimal", SymDecimal, sym.Decimal},
				{"Group", SymGroup, sym.Group},
				{"List", SymList, sym.List},
				{"PercentSign", SymPercentSign, sym.PercentSign},
				{"PlusSign", SymPlusSign, sym.PlusSign},
				{"MinusSign", SymMinusSign, sym.MinusSign},
				{"Exponential", SymExponential, sym.Exponential},
				{"SuperscriptingExponent", SymSuperscriptingExponent, sym.SuperscriptingExponent},
				{"PerMille", SymPerMille, sym.PerMille},
				{"Infinity", SymInfinity, sym.Infinity},
				{"NaN", SymNan, sym.Nan},
				{"TimeSeparator", SymTimeSeparator, sym.TimeSeparator},
			}
			info := InfoFromLangID(langIndex, sym.NumberSystem)
			for _, tc := range testCases {
				// Extract the wanted value.
				v := reflect.ValueOf(tc.x)
				if v.Len() == 0 {
					return
				}
				if v.Len() > 1 {
					t.Fatalf("Multiple values of %q within single symbol not supported.", tc.name)
				}
				want := v.Index(0).MethodByName("Data").Call(nil)[0].String()
				got := info.Symbol(tc.st)
				if got != want {
					t.Errorf("%s:%s:%s: got %q; want %q", lang, sym.NumberSystem, tc.name, got, want)
				}
			}
		}
	}
}
func skipShort(t *testing.T) {
	testtext.SkipIfNotLong(t)

	once.Do(func() { loadTestData(t) })
}