func TestSingleCharacterScannerCanProcess(t *testing.T) {
testCaseList := []testCaseForTestSingleCharacterScannerCanProcess{
{data: "{", tokenKind: token.KindBeginObject, expected: true},
{data: "}", tokenKind: token.KindEndObject, expected: true},
{data: "[", tokenKind: token.KindBeginArray, expected: true},
{data: "]", tokenKind: token.KindEndArray, expected: true},
{data: ",", tokenKind: token.KindComma, expected: true},
{data: ":", tokenKind: token.KindSeparator, expected: true},
{data: "", tokenKind: token.KindBeginObject, expected: false},
{data: "a", tokenKind: token.KindEndObject, expected: false},
{data: "}", tokenKind: token.KindBeginArray, expected: false},
{data: "{", tokenKind: token.KindEndArray, expected: false},
{data: "-", tokenKind: token.KindComma, expected: false},
{data: "=", tokenKind: token.KindSeparator, expected: false},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
actual := scanner.CanProcess()
if actual != testCase.expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
}
}
}
func TestSingleCharacterScannerRuneOfToken(t *testing.T) {
testCaseList := []testCaseForTestSingleCharacterScannerRuneOfToken{
{data: "{", tokenKind: token.KindBeginObject, expected: '{'},
{data: "}", tokenKind: token.KindEndObject, expected: '}'},
{data: "[", tokenKind: token.KindBeginArray, expected: '['},
{data: "]", tokenKind: token.KindEndArray, expected: ']'},
{data: ",", tokenKind: token.KindComma, expected: ','},
{data: ":", tokenKind: token.KindSeparator, expected: ':'},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
actual := scanner.(*tokenscanner.SingleCharacterScanner).RuneOfToken()
if actual != testCase.expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
}
}
}
func TestSingleCharacterScannerMultiScanner(t *testing.T) {
testData := `
{
[ ],
:
}
`
expectedString := `{[],:}`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
}
var buffer bytes.Buffer
for {
_, err := jsonScanner.PeekNonWhiteSpaceRune()
if err == io.EOF {
break
}
scanned := false
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
scanned = true
currentToken, err := scanner.Scan()
buffer.WriteString(currentToken.Value)
if err != nil {
t.Log(scannerIndex)
t.Fatal(err)
}
}
}
if !scanned {
peek, err := jsonScanner.PeekNonWhiteSpaceRune()
if err != nil {
t.Log("err:", err)
} else {
t.Log("current rune:", string(peek))
}
t.Fatal("can't scan")
}
}
if buffer.String() != expectedString {
t.Errorf("Scan result is wrong expected=%#v, actual=%#v\n", testData, buffer.String())
}
}
func createJSONScanners(scanner scanner.Scanner) []tokenscanner.TokenScanner {
return []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, scanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, scanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, scanner),
tokenscanner.NewStringScanner(scanner),
tokenscanner.NewNumberScanner(scanner),
}
}
func TestSingleCharacterScannerScan(t *testing.T) {
testCaseList := []testCaseForTestSingleCharacterScannerScan{
{data: "{", tokenKind: token.KindBeginObject, expected: token.NewToken(token.KindBeginObject, string(token.KindBeginObject))},
{data: "}", tokenKind: token.KindEndObject, expected: token.NewToken(token.KindEndObject, string(token.KindEndObject))},
{data: "[", tokenKind: token.KindBeginArray, expected: token.NewToken(token.KindBeginArray, string(token.KindBeginArray))},
{data: "]", tokenKind: token.KindEndArray, expected: token.NewToken(token.KindEndArray, string(token.KindEndArray))},
{data: ",", tokenKind: token.KindComma, expected: token.NewToken(token.KindComma, string(token.KindComma))},
{data: ":", tokenKind: token.KindSeparator, expected: token.NewToken(token.KindSeparator, string(token.KindSeparator))},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
actual, err := scanner.Scan()
if err != nil {
t.Logf("testCase[%d]", testIndex)
t.Error(err)
}
if actual != testCase.expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
}
}
}
func TestCompositeCase(t *testing.T) {
testData := `
{
false: [true, null,
{
null: true
}]
}
`
expectedString := "{false:[true,null,{null:true}]}"
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
}
var buffer bytes.Buffer
expectedTokenIndex := 0
for {
_, err := jsonScanner.PeekNonWhiteSpaceRune()
if err == io.EOF {
break
}
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
scannedToken, err := scanner.Scan()
if err != nil {
t.Log(scannerIndex)
t.Error(err)
}
if !expectedTokens[expectedTokenIndex].IsEqualTo(scannedToken) {
t.Errorf("expected=%#v, actual=%#v", expectedTokens[expectedTokenIndex], scannedToken)
}
expectedTokenIndex++
buffer.WriteString(scannedToken.Value)
}
}
}
if buffer.String() != expectedString {
t.Errorf("expected=%#v, actual=%#v", expectedString, buffer.String())
}
}
func TestTokenizer1(t *testing.T) {
testData := `
{
"key": [
{
"inner1": "stringValue",
"inner2": true,
"inner3": false,
"inner4": null,
"inner5": {
"nested": -2.5e-3
}
}
]
}
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewStringScanner(jsonScanner),
tokenscanner.NewNumberScanner(jsonScanner),
}
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "key"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "inner1"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindString, "stringValue"),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner2"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner3"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner4"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner5"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "nested"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNumber, "-2.5e-3"),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
for {
if _, err := jsonScanner.PeekNonWhiteSpaceRune(); err == io.EOF {
break
}
for expectedIndex, expected := range expectedTokens {
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
actual, err := scanner.Scan()
if err != nil {
t.Logf("scanners[%d]: ", scannerIndex)
t.Error(err)
}
if !expected.IsEqualTo(actual) {
t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
}
break
}
}
}
}
}