func TestStringScanner1(t *testing.T) {
testData := `
"teststring"
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewStringScanner(jsonScanner).(*tokenscanner.StringScanner)
if !scanner.CanProcess() {
t.Error("testData can process")
}
if scanner.RuneOfToken() != '"' {
t.Error("StringScanner.RunOfToken must be '\"'")
}
if !scanner.CanProcessRune('"') {
t.Error("StringScanner.CanProcessRune can process rune '\"'")
}
expectedToken := token.NewToken(token.KindString, "teststring")
token, err := scanner.Scan()
if err != nil {
t.Error(err)
}
if !expectedToken.IsEqualTo(token) {
t.Errorf("expected=%#v, actual=%#v", expectedToken, token)
}
}
func createJSONScanners(scanner scanner.Scanner) []tokenscanner.TokenScanner {
return []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, scanner),
tokenscanner.NewStringScanner(scanner),
tokenscanner.NewNumberScanner(scanner),
}
}
func TestStringScannerScan(t *testing.T) {
testCaseList := []testCaseForTestStringScannerScan{
{
data: `
"abcdefg"
`,
expected: token.NewToken(token.KindString, "abcdefg"),
},
{
data: `
"\t\n\r\b\f\\\/"
`,
expected: token.NewToken(token.KindString,
` `+`
`+"\r\b\f\\/"),
},
{
data: `
"\u65E5\u672C\u8A9E\uD867\uDE3D"
`,
expected: token.NewToken(token.KindString, "日本語𩸽"),
},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewStringScanner(jsonScanner).(*tokenscanner.StringScanner)
actual, err := scanner.Scan()
if err != nil {
t.Error(err)
}
if !testCase.expected.IsEqualTo(actual) {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, testCase.expected, actual)
}
}
}
func TestTokenizer1(t *testing.T) {
testData := `
{
"key": [
{
"inner1": "stringValue",
"inner2": true,
"inner3": false,
"inner4": null,
"inner5": {
"nested": -2.5e-3
}
}
]
}
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewStringScanner(jsonScanner),
tokenscanner.NewNumberScanner(jsonScanner),
}
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "key"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "inner1"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindString, "stringValue"),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner2"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner3"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner4"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner5"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "nested"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNumber, "-2.5e-3"),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
for {
if _, err := jsonScanner.PeekNonWhiteSpaceRune(); err == io.EOF {
break
}
for expectedIndex, expected := range expectedTokens {
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
actual, err := scanner.Scan()
if err != nil {
t.Logf("scanners[%d]: ", scannerIndex)
t.Error(err)
}
if !expected.IsEqualTo(actual) {
t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
}
break
}
}
}
}
}