func TestMultiCharacterScannerScan(t *testing.T) {
testCaseList := []testCaseForTestMultiCharacterScannerScan{
{data: " null ", tokenKind: token.KindNull, expectedError: false, expected: token.NewToken(token.KindNull, string(token.KindNull))},
{data: " true ", tokenKind: token.KindTrue, expectedError: false, expected: token.NewToken(token.KindTrue, string(token.KindTrue))},
{data: "false ", tokenKind: token.KindFalse, expectedError: false, expected: token.NewToken(token.KindFalse, string(token.KindFalse))},
{data: "f alse ", tokenKind: token.KindFalse, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
{data: "tru e", tokenKind: token.KindTrue, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
{data: "xnull", tokenKind: token.KindNull, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
}
for testIndex, testCase := range testCaseList {
stringReader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(stringReader)
scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)
actual, err := scanner.Scan()
if err == nil && testCase.expectedError {
t.Errorf("testCase[%d] must return error", testIndex)
}
if err != nil && !testCase.expectedError {
t.Error(err)
}
expected := testCase.expected
if !actual.IsEqualTo(expected) {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, expected, actual)
}
}
}
func createJSONScanners(scanner scanner.Scanner) []tokenscanner.TokenScanner {
return []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, scanner),
tokenscanner.NewStringScanner(scanner),
tokenscanner.NewNumberScanner(scanner),
}
}
func TestMultiCharacterScannerRuneOfToken(t *testing.T) {
testCaseList := []testCaseForTestMultiCharacterScannerRuneOfToken{
{data: " null ", tokenKind: token.KindNull, expected: 'n'},
{data: " true ", tokenKind: token.KindTrue, expected: 't'},
{data: " false ", tokenKind: token.KindFalse, expected: 'f'},
}
for testIndex, testCase := range testCaseList {
stringReader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(stringReader)
scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)
actual := scanner.RuneOfToken()
expected := testCase.expected
if actual != expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, string(expected), string(actual))
}
}
}
func TestMultiCharacterScannerCanProcess(t *testing.T) {
testCaseList := []testCaseForTestMultiCharacterScannerCanProcess{
{data: " null ", tokenKind: token.KindNull, expected: true},
{data: " true ", tokenKind: token.KindTrue, expected: true},
{data: "false ", tokenKind: token.KindFalse, expected: true},
{data: "xtrue ", tokenKind: token.KindFalse, expected: false},
{data: "alse ", tokenKind: token.KindTrue, expected: false},
{data: "false ", tokenKind: token.KindNull, expected: false},
}
for testIndex, testCase := range testCaseList {
stringReader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(stringReader)
scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)
actual := scanner.CanProcess()
expected := testCase.expected
if actual != expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, expected, actual)
}
}
}
func TestCompositeCase(t *testing.T) {
testData := `
{
false: [true, null,
{
null: true
}]
}
`
expectedString := "{false:[true,null,{null:true}]}"
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
}
var buffer bytes.Buffer
expectedTokenIndex := 0
for {
_, err := jsonScanner.PeekNonWhiteSpaceRune()
if err == io.EOF {
break
}
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
scannedToken, err := scanner.Scan()
if err != nil {
t.Log(scannerIndex)
t.Error(err)
}
if !expectedTokens[expectedTokenIndex].IsEqualTo(scannedToken) {
t.Errorf("expected=%#v, actual=%#v", expectedTokens[expectedTokenIndex], scannedToken)
}
expectedTokenIndex++
buffer.WriteString(scannedToken.Value)
}
}
}
if buffer.String() != expectedString {
t.Errorf("expected=%#v, actual=%#v", expectedString, buffer.String())
}
}
func TestTokenizer1(t *testing.T) {
testData := `
{
"key": [
{
"inner1": "stringValue",
"inner2": true,
"inner3": false,
"inner4": null,
"inner5": {
"nested": -2.5e-3
}
}
]
}
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewStringScanner(jsonScanner),
tokenscanner.NewNumberScanner(jsonScanner),
}
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "key"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "inner1"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindString, "stringValue"),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner2"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner3"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner4"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner5"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "nested"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNumber, "-2.5e-3"),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
for {
if _, err := jsonScanner.PeekNonWhiteSpaceRune(); err == io.EOF {
break
}
for expectedIndex, expected := range expectedTokens {
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
actual, err := scanner.Scan()
if err != nil {
t.Logf("scanners[%d]: ", scannerIndex)
t.Error(err)
}
if !expected.IsEqualTo(actual) {
t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
}
break
}
}
}
}
}