func TestJSONScannerNextCurrentPosition(t *testing.T) {
testCaseList := []testCaseForJSONScannerCurrentPosition{
{data: "12345678", expected: []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
}
for testIndex, testCase := range testCaseList {
scanner := scanner.NewJSONScanner(strings.NewReader(testCase.data))
for expectedIndex, expected := range testCase.expected {
actual := scanner.CurrentPosition()
if actual != expected {
t.Errorf("testCase[%d:%d] expected=%#v, actual=%#v", testIndex, expectedIndex, expected, actual)
}
scanner.NextRune()
}
actual, err := scanner.NextNonWhiteSpaceRune()
if err != io.EOF {
t.Logf("%s", string([]rune{actual}))
t.Errorf("testCase[%d] EOF not found", testIndex)
}
actual, err = scanner.NextRune()
if err != io.EOF {
t.Logf("%s", string([]rune{actual}))
t.Errorf("testCase[%d] EOF not found, current:%d", testIndex, scanner.CurrentPosition())
}
}
}
func TestMultiCharacterScannerScan(t *testing.T) {
testCaseList := []testCaseForTestMultiCharacterScannerScan{
{data: " null ", tokenKind: token.KindNull, expectedError: false, expected: token.NewToken(token.KindNull, string(token.KindNull))},
{data: " true ", tokenKind: token.KindTrue, expectedError: false, expected: token.NewToken(token.KindTrue, string(token.KindTrue))},
{data: "false ", tokenKind: token.KindFalse, expectedError: false, expected: token.NewToken(token.KindFalse, string(token.KindFalse))},
{data: "f alse ", tokenKind: token.KindFalse, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
{data: "tru e", tokenKind: token.KindTrue, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
{data: "xnull", tokenKind: token.KindNull, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
}
for testIndex, testCase := range testCaseList {
stringReader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(stringReader)
scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)
actual, err := scanner.Scan()
if err == nil && testCase.expectedError {
t.Errorf("testCase[%d] must return error", testIndex)
}
if err != nil && !testCase.expectedError {
t.Error(err)
}
expected := testCase.expected
if !actual.IsEqualTo(expected) {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, expected, actual)
}
}
}
func TestStringScanner1(t *testing.T) {
testData := `
"teststring"
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewStringScanner(jsonScanner).(*tokenscanner.StringScanner)
if !scanner.CanProcess() {
t.Error("testData can process")
}
if scanner.RuneOfToken() != '"' {
t.Error("StringScanner.RunOfToken must be '\"'")
}
if !scanner.CanProcessRune('"') {
t.Error("StringScanner.CanProcessRune can process rune '\"'")
}
expectedToken := token.NewToken(token.KindString, "teststring")
token, err := scanner.Scan()
if err != nil {
t.Error(err)
}
if !expectedToken.IsEqualTo(token) {
t.Errorf("expected=%#v, actual=%#v", expectedToken, token)
}
}
func TestSingleCharacterScannerCanProcess(t *testing.T) {
testCaseList := []testCaseForTestSingleCharacterScannerCanProcess{
{data: "{", tokenKind: token.KindBeginObject, expected: true},
{data: "}", tokenKind: token.KindEndObject, expected: true},
{data: "[", tokenKind: token.KindBeginArray, expected: true},
{data: "]", tokenKind: token.KindEndArray, expected: true},
{data: ",", tokenKind: token.KindComma, expected: true},
{data: ":", tokenKind: token.KindSeparator, expected: true},
{data: "", tokenKind: token.KindBeginObject, expected: false},
{data: "a", tokenKind: token.KindEndObject, expected: false},
{data: "}", tokenKind: token.KindBeginArray, expected: false},
{data: "{", tokenKind: token.KindEndArray, expected: false},
{data: "-", tokenKind: token.KindComma, expected: false},
{data: "=", tokenKind: token.KindSeparator, expected: false},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
actual := scanner.CanProcess()
if actual != testCase.expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
}
}
}
func TestJSONScannerNextPeekRune(t *testing.T) {
testCaseList := []testCaseForJSONScannerPeekRune{
{data: "12345678", expected: "12345678"},
}
for testIndex, testCase := range testCaseList {
scanner := scanner.NewJSONScanner(strings.NewReader(testCase.data))
for expectedIndex, expected := range testCase.expected {
beforePosition := scanner.CurrentPosition()
actual, err := scanner.PeekRune()
afterPosition := scanner.CurrentPosition()
if err != nil {
t.Error(err)
}
if actual != expected {
t.Errorf("testCase[%d:%d] expected=%#v, actual=%#v", testIndex, expectedIndex, expected, actual)
}
if beforePosition != afterPosition {
t.Errorf("testCase[%d:%d] position is wrong", testIndex, expectedIndex)
}
scanner.NextRune()
}
actual, err := scanner.NextNonWhiteSpaceRune()
if err != io.EOF {
t.Logf("%s", string([]rune{actual}))
t.Errorf("testCase[%d] EOF not found", testIndex)
}
actual, err = scanner.NextRune()
if err != io.EOF {
t.Logf("%s", string([]rune{actual}))
t.Errorf("testCase[%d] EOF not found, current:%d", testIndex, scanner.CurrentPosition())
}
}
}
// NewTokenizer returns a new tokenizer.
func NewTokenizer(reader io.Reader) *Tokenizer {
jsonScanner := scanner.NewJSONScanner(reader)
return &Tokenizer{
scanner: jsonScanner,
tokenScanners: createJSONScanners(jsonScanner),
buffer: &tokenBuffer{buffer: []token.Token{}},
}
}
func TestJSONScannerNewScanner(t *testing.T) {
testData := `
{
"key": "value"
}
`
actual := scanner.NewJSONScanner(strings.NewReader(testData))
if actual == nil {
t.Error("scanner not created")
}
}
func TestSingleCharacterScannerMultiScanner(t *testing.T) {
testData := `
{
[ ],
:
}
`
expectedString := `{[],:}`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
}
var buffer bytes.Buffer
for {
_, err := jsonScanner.PeekNonWhiteSpaceRune()
if err == io.EOF {
break
}
scanned := false
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
scanned = true
currentToken, err := scanner.Scan()
buffer.WriteString(currentToken.Value)
if err != nil {
t.Log(scannerIndex)
t.Fatal(err)
}
}
}
if !scanned {
peek, err := jsonScanner.PeekNonWhiteSpaceRune()
if err != nil {
t.Log("err:", err)
} else {
t.Log("current rune:", string(peek))
}
t.Fatal("can't scan")
}
}
if buffer.String() != expectedString {
t.Errorf("Scan result is wrong expected=%#v, actual=%#v\n", testData, buffer.String())
}
}
func TestSingleCharacterScannerRuneOfToken(t *testing.T) {
testCaseList := []testCaseForTestSingleCharacterScannerRuneOfToken{
{data: "{", tokenKind: token.KindBeginObject, expected: '{'},
{data: "}", tokenKind: token.KindEndObject, expected: '}'},
{data: "[", tokenKind: token.KindBeginArray, expected: '['},
{data: "]", tokenKind: token.KindEndArray, expected: ']'},
{data: ",", tokenKind: token.KindComma, expected: ','},
{data: ":", tokenKind: token.KindSeparator, expected: ':'},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
actual := scanner.(*tokenscanner.SingleCharacterScanner).RuneOfToken()
if actual != testCase.expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
}
}
}
func TestMultiCharacterScannerRuneOfToken(t *testing.T) {
testCaseList := []testCaseForTestMultiCharacterScannerRuneOfToken{
{data: " null ", tokenKind: token.KindNull, expected: 'n'},
{data: " true ", tokenKind: token.KindTrue, expected: 't'},
{data: " false ", tokenKind: token.KindFalse, expected: 'f'},
}
for testIndex, testCase := range testCaseList {
stringReader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(stringReader)
scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)
actual := scanner.RuneOfToken()
expected := testCase.expected
if actual != expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, string(expected), string(actual))
}
}
}
func TestJSONScannerNextPeekRuneWithOffset(t *testing.T) {
testCaseList := []testCaseForJSONScannerPeekRuneWithOffset{
{data: "12345678", expected: []rune{'1', '2', '3', '4', '5', '6', '7', '8'}},
}
for testIndex, testCase := range testCaseList {
scanner := scanner.NewJSONScanner(strings.NewReader(testCase.data))
index := 0
for expectedIndex, expected := range testCase.expected {
actual, err := scanner.PeekRuneWithOffset(index)
index++
if err != nil {
t.Fatal(err)
}
if actual != expected {
t.Errorf("testCase[%d:%d] expected=%#v, actual=%#v", testIndex, expectedIndex, expected, actual)
}
}
for expectedIndex, expected := range testCase.expected {
actual, err := scanner.NextRune()
if err != nil {
t.Fatal(err)
}
if actual != expected {
t.Errorf("testCase[%d:%d] expected=%#v, actual=%#v", testIndex, expectedIndex, expected, actual)
}
}
actual, err := scanner.NextNonWhiteSpaceRune()
if err != io.EOF {
t.Logf("%s", string([]rune{actual}))
t.Errorf("testCase[%d] EOF not found", testIndex)
}
actual, err = scanner.NextRune()
if err != io.EOF {
t.Logf("%s", string([]rune{actual}))
t.Errorf("testCase[%d] EOF not found, current:%d", testIndex, scanner.CurrentPosition())
}
}
}
func TestNumber1(t *testing.T) {
testData := `
-2.8e-10
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewNumberScanner(jsonScanner)
if !scanner.CanProcess() {
t.Error("token can't process")
}
scannedToken, err := scanner.Scan()
if err != nil {
t.Error(err)
}
if scannedToken.Kind != token.KindNumber {
t.Error("token must be number")
}
if scannedToken.Value != "-2.8e-10" {
t.Errorf("token value is wrong: %#v", scannedToken.Value)
}
}
func TestMultiCharacterScannerCanProcess(t *testing.T) {
testCaseList := []testCaseForTestMultiCharacterScannerCanProcess{
{data: " null ", tokenKind: token.KindNull, expected: true},
{data: " true ", tokenKind: token.KindTrue, expected: true},
{data: "false ", tokenKind: token.KindFalse, expected: true},
{data: "xtrue ", tokenKind: token.KindFalse, expected: false},
{data: "alse ", tokenKind: token.KindTrue, expected: false},
{data: "false ", tokenKind: token.KindNull, expected: false},
}
for testIndex, testCase := range testCaseList {
stringReader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(stringReader)
scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)
actual := scanner.CanProcess()
expected := testCase.expected
if actual != expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, expected, actual)
}
}
}
func TestJSONScannerNextRune(t *testing.T) {
testCaseList := []testCaseForJSONScannerNextRune{
{data: "12345678"},
{data: "abcd efghij"},
}
for testIndex, testCase := range testCaseList {
scanner := scanner.NewJSONScanner(strings.NewReader(testCase.data))
for expectedIndex, expected := range testCase.data {
actual, err := scanner.NextRune()
if err != nil {
t.Error(err)
}
if actual != expected {
t.Errorf("testCase[%d:%d] expected=%#v, actual=%#v", testIndex, expectedIndex, expected, actual)
}
}
_, err := scanner.NextRune()
if err != io.EOF {
t.Errorf("testCase[%d] EOF not found", testIndex)
}
}
}
func TestSingleCharacterScannerScan(t *testing.T) {
testCaseList := []testCaseForTestSingleCharacterScannerScan{
{data: "{", tokenKind: token.KindBeginObject, expected: token.NewToken(token.KindBeginObject, string(token.KindBeginObject))},
{data: "}", tokenKind: token.KindEndObject, expected: token.NewToken(token.KindEndObject, string(token.KindEndObject))},
{data: "[", tokenKind: token.KindBeginArray, expected: token.NewToken(token.KindBeginArray, string(token.KindBeginArray))},
{data: "]", tokenKind: token.KindEndArray, expected: token.NewToken(token.KindEndArray, string(token.KindEndArray))},
{data: ",", tokenKind: token.KindComma, expected: token.NewToken(token.KindComma, string(token.KindComma))},
{data: ":", tokenKind: token.KindSeparator, expected: token.NewToken(token.KindSeparator, string(token.KindSeparator))},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
actual, err := scanner.Scan()
if err != nil {
t.Logf("testCase[%d]", testIndex)
t.Error(err)
}
if actual != testCase.expected {
t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
}
}
}
func TestStringScannerScan(t *testing.T) {
testCaseList := []testCaseForTestStringScannerScan{
{
data: `
"abcdefg"
`,
expected: token.NewToken(token.KindString, "abcdefg"),
},
{
data: `
"\t\n\r\b\f\\\/"
`,
expected: token.NewToken(token.KindString,
` `+`
`+"\r\b\f\\/"),
},
{
data: `
"\u65E5\u672C\u8A9E\uD867\uDE3D"
`,
expected: token.NewToken(token.KindString, "日本語𩸽"),
},
}
for testIndex, testCase := range testCaseList {
reader := strings.NewReader(testCase.data)
jsonScanner := scanner.NewJSONScanner(reader)
scanner := tokenscanner.NewStringScanner(jsonScanner).(*tokenscanner.StringScanner)
actual, err := scanner.Scan()
if err != nil {
t.Error(err)
}
if !testCase.expected.IsEqualTo(actual) {
t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, testCase.expected, actual)
}
}
}
func TestTokenizer1(t *testing.T) {
testData := `
{
"key": [
{
"inner1": "stringValue",
"inner2": true,
"inner3": false,
"inner4": null,
"inner5": {
"nested": -2.5e-3
}
}
]
}
`
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewStringScanner(jsonScanner),
tokenscanner.NewNumberScanner(jsonScanner),
}
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "key"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "inner1"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindString, "stringValue"),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner2"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner3"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner4"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindString, "inner5"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindString, "nested"),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindNumber, "-2.5e-3"),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
for {
if _, err := jsonScanner.PeekNonWhiteSpaceRune(); err == io.EOF {
break
}
for expectedIndex, expected := range expectedTokens {
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
actual, err := scanner.Scan()
if err != nil {
t.Logf("scanners[%d]: ", scannerIndex)
t.Error(err)
}
if !expected.IsEqualTo(actual) {
t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
}
break
}
}
}
}
}
func TestCompositeCase(t *testing.T) {
testData := `
{
false: [true, null,
{
null: true
}]
}
`
expectedString := "{false:[true,null,{null:true}]}"
expectedTokens := []token.Token{
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindFalse, string(token.KindFalse)),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindComma, string(token.KindComma)),
token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
token.NewToken(token.KindNull, string(token.KindNull)),
token.NewToken(token.KindSeparator, string(token.KindSeparator)),
token.NewToken(token.KindTrue, string(token.KindTrue)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
token.NewToken(token.KindEndArray, string(token.KindEndArray)),
token.NewToken(token.KindEndObject, string(token.KindEndObject)),
}
reader := strings.NewReader(testData)
jsonScanner := scanner.NewJSONScanner(reader)
scanners := []tokenscanner.TokenScanner{
tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
}
var buffer bytes.Buffer
expectedTokenIndex := 0
for {
_, err := jsonScanner.PeekNonWhiteSpaceRune()
if err == io.EOF {
break
}
for scannerIndex, scanner := range scanners {
if scanner.CanProcess() {
scannedToken, err := scanner.Scan()
if err != nil {
t.Log(scannerIndex)
t.Error(err)
}
if !expectedTokens[expectedTokenIndex].IsEqualTo(scannedToken) {
t.Errorf("expected=%#v, actual=%#v", expectedTokens[expectedTokenIndex], scannedToken)
}
expectedTokenIndex++
buffer.WriteString(scannedToken.Value)
}
}
}
if buffer.String() != expectedString {
t.Errorf("expected=%#v, actual=%#v", expectedString, buffer.String())
}
}