示例#1
0
func TestStringScanner1(t *testing.T) {
	testData := `
	"teststring"
	`
	reader := strings.NewReader(testData)
	jsonScanner := scanner.NewJSONScanner(reader)
	scanner := tokenscanner.NewStringScanner(jsonScanner).(*tokenscanner.StringScanner)
	if !scanner.CanProcess() {
		t.Error("testData can process")
	}
	if scanner.RuneOfToken() != '"' {
		t.Error("StringScanner.RunOfToken must be '\"'")
	}
	if !scanner.CanProcessRune('"') {
		t.Error("StringScanner.CanProcessRune can process rune '\"'")
	}

	expectedToken := token.NewToken(token.KindString, "teststring")
	token, err := scanner.Scan()
	if err != nil {
		t.Error(err)
	}
	if !expectedToken.IsEqualTo(token) {
		t.Errorf("expected=%#v, actual=%#v", expectedToken, token)
	}
}
示例#2
0
func TestMultiCharacterScannerScan(t *testing.T) {
	testCaseList := []testCaseForTestMultiCharacterScannerScan{
		{data: " null  ", tokenKind: token.KindNull, expectedError: false, expected: token.NewToken(token.KindNull, string(token.KindNull))},
		{data: " true  ", tokenKind: token.KindTrue, expectedError: false, expected: token.NewToken(token.KindTrue, string(token.KindTrue))},
		{data: "false ", tokenKind: token.KindFalse, expectedError: false, expected: token.NewToken(token.KindFalse, string(token.KindFalse))},

		{data: "f alse ", tokenKind: token.KindFalse, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
		{data: "tru e", tokenKind: token.KindTrue, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
		{data: "xnull", tokenKind: token.KindNull, expectedError: true, expected: token.NewToken(token.KindEmpty, "")},
	}
	for testIndex, testCase := range testCaseList {
		stringReader := strings.NewReader(testCase.data)
		jsonScanner := scanner.NewJSONScanner(stringReader)
		scanner := tokenscanner.NewMultiCharacterScanner(testCase.tokenKind, jsonScanner).(*tokenscanner.MultiCharacterScanner)

		actual, err := scanner.Scan()
		if err == nil && testCase.expectedError {
			t.Errorf("testCase[%d] must return error", testIndex)
		}
		if err != nil && !testCase.expectedError {
			t.Error(err)
		}
		expected := testCase.expected

		if !actual.IsEqualTo(expected) {
			t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, expected, actual)
		}
	}
}
示例#3
0
func TestStringScannerScan(t *testing.T) {
	testCaseList := []testCaseForTestStringScannerScan{
		{
			data: `
		"abcdefg"
		`,
			expected: token.NewToken(token.KindString, "abcdefg"),
		},
		{
			data: `
		"\t\n\r\b\f\\\/"
		`,
			expected: token.NewToken(token.KindString,
				`	`+`
`+"\r\b\f\\/"),
		},
		{
			data: `
		"\u65E5\u672C\u8A9E\uD867\uDE3D"
		`,
			expected: token.NewToken(token.KindString, "日本語𩸽"),
		},
	}
	for testIndex, testCase := range testCaseList {
		reader := strings.NewReader(testCase.data)
		jsonScanner := scanner.NewJSONScanner(reader)
		scanner := tokenscanner.NewStringScanner(jsonScanner).(*tokenscanner.StringScanner)
		actual, err := scanner.Scan()
		if err != nil {
			t.Error(err)
		}
		if !testCase.expected.IsEqualTo(actual) {
			t.Errorf("testCase[%d] expected=%#v, actual=%#v", testIndex, testCase.expected, actual)
		}
	}
}
示例#4
0
func TestSingleCharacterScannerScan(t *testing.T) {
	testCaseList := []testCaseForTestSingleCharacterScannerScan{
		{data: "{", tokenKind: token.KindBeginObject, expected: token.NewToken(token.KindBeginObject, string(token.KindBeginObject))},
		{data: "}", tokenKind: token.KindEndObject, expected: token.NewToken(token.KindEndObject, string(token.KindEndObject))},
		{data: "[", tokenKind: token.KindBeginArray, expected: token.NewToken(token.KindBeginArray, string(token.KindBeginArray))},
		{data: "]", tokenKind: token.KindEndArray, expected: token.NewToken(token.KindEndArray, string(token.KindEndArray))},
		{data: ",", tokenKind: token.KindComma, expected: token.NewToken(token.KindComma, string(token.KindComma))},
		{data: ":", tokenKind: token.KindSeparator, expected: token.NewToken(token.KindSeparator, string(token.KindSeparator))},
	}

	for testIndex, testCase := range testCaseList {
		reader := strings.NewReader(testCase.data)
		jsonScanner := scanner.NewJSONScanner(reader)
		scanner := tokenscanner.NewSingleCharacterScanner(testCase.tokenKind, jsonScanner)
		actual, err := scanner.Scan()
		if err != nil {
			t.Logf("testCase[%d]", testIndex)
			t.Error(err)
		}
		if actual != testCase.expected {
			t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, testCase.expected, actual)
		}
	}
}
示例#5
0
func TestTokenIsSameKindTo(t *testing.T) {
	testCaseList := []testCaseForTestTokenIsSameKindTo{
		{
			left:     token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
			right:    token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
			expected: true,
		},
		{
			left:     token.NewToken(token.KindEndObject, string(token.KindEndObject)),
			right:    token.NewToken(token.KindEndObject, string(token.KindEndObject)),
			expected: true,
		},
		{
			left:     token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
			right:    token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
			expected: true,
		},
		{
			left:     token.NewToken(token.KindEndArray, string(token.KindEndArray)),
			right:    token.NewToken(token.KindEndArray, string(token.KindEndArray)),
			expected: true,
		},
		{
			left:     token.NewToken(token.KindComma, string(token.KindComma)),
			right:    token.NewToken(token.KindComma, string(token.KindComma)),
			expected: true,
		},
		{
			left:     token.NewToken(token.KindSeparator, string(token.KindSeparator)),
			right:    token.NewToken(token.KindSeparator, string(token.KindSeparator)),
			expected: true,
		},
		// Kinds are same, Values are not equal.
		{
			left:     token.NewToken(token.KindString, "test1"),
			right:    token.NewToken(token.KindString, "test2"),
			expected: true,
		},
		// Kinds are not equal , Values are equal.
		{
			left:     token.NewToken(token.KindString, "test1"),
			right:    token.NewToken(token.KindNumber, "test1"),
			expected: false,
		},
	}
	for testIndex, testCase := range testCaseList {
		left := testCase.left
		right := testCase.right
		actual := left.IsSameKindTo(right)
		expected := testCase.expected
		if actual != expected {
			t.Errorf("testCase[%d] expected=%#v, actual=%#v\n", testIndex, expected, actual)
		}
		if left.IsSameKindTo(right) != right.IsSameKindTo(left) {
			t.Errorf("testCase[%d] IsSameKindTo method must be obey commutative property", testIndex)
		}
	}
}
示例#6
0
func TestCompositeCase(t *testing.T) {
	testData := `
	{
		false: [true, null,
		{
			null: true
		}]
	}
	`
	expectedString := "{false:[true,null,{null:true}]}"
	expectedTokens := []token.Token{
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
		token.NewToken(token.KindFalse, string(token.KindFalse)),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
		token.NewToken(token.KindTrue, string(token.KindTrue)),
		token.NewToken(token.KindComma, string(token.KindComma)),
		token.NewToken(token.KindNull, string(token.KindNull)),
		token.NewToken(token.KindComma, string(token.KindComma)),
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
		token.NewToken(token.KindNull, string(token.KindNull)),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindTrue, string(token.KindTrue)),
		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
		token.NewToken(token.KindEndArray, string(token.KindEndArray)),
		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
	}

	reader := strings.NewReader(testData)
	jsonScanner := scanner.NewJSONScanner(reader)
	scanners := []tokenscanner.TokenScanner{
		tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),
		tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),
		tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
		tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
	}
	var buffer bytes.Buffer
	expectedTokenIndex := 0
	for {
		_, err := jsonScanner.PeekNonWhiteSpaceRune()
		if err == io.EOF {
			break
		}
		for scannerIndex, scanner := range scanners {
			if scanner.CanProcess() {
				scannedToken, err := scanner.Scan()
				if err != nil {
					t.Log(scannerIndex)
					t.Error(err)
				}
				if !expectedTokens[expectedTokenIndex].IsEqualTo(scannedToken) {
					t.Errorf("expected=%#v, actual=%#v", expectedTokens[expectedTokenIndex], scannedToken)
				}
				expectedTokenIndex++
				buffer.WriteString(scannedToken.Value)
			}
		}
	}
	if buffer.String() != expectedString {
		t.Errorf("expected=%#v, actual=%#v", expectedString, buffer.String())
	}
}
示例#7
0
func TestTokenizer1(t *testing.T) {
	testData := `
	{
		"key": [
			{
				"inner1": "stringValue",
				"inner2": true,
				"inner3": false,
				"inner4": null,
				"inner5": {
					"nested": -2.5e-3
				}
			}
		]
	}
	`
	reader := strings.NewReader(testData)
	jsonScanner := scanner.NewJSONScanner(reader)

	scanners := []tokenscanner.TokenScanner{
		tokenscanner.NewSingleCharacterScanner(token.KindBeginObject, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindEndObject, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindBeginArray, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindEndArray, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindSeparator, jsonScanner),
		tokenscanner.NewSingleCharacterScanner(token.KindComma, jsonScanner),

		tokenscanner.NewMultiCharacterScanner(token.KindTrue, jsonScanner),
		tokenscanner.NewMultiCharacterScanner(token.KindFalse, jsonScanner),
		tokenscanner.NewMultiCharacterScanner(token.KindNull, jsonScanner),

		tokenscanner.NewStringScanner(jsonScanner),
		tokenscanner.NewNumberScanner(jsonScanner),
	}

	expectedTokens := []token.Token{
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
		token.NewToken(token.KindString, "key"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),

		token.NewToken(token.KindString, "inner1"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindString, "stringValue"),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner2"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindTrue, string(token.KindTrue)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner3"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindFalse, string(token.KindFalse)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner4"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindNull, string(token.KindNull)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner5"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),

		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),

		token.NewToken(token.KindString, "nested"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindNumber, "-2.5e-3"),

		token.NewToken(token.KindEndObject, string(token.KindEndObject)),

		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
		token.NewToken(token.KindEndArray, string(token.KindEndArray)),
		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
	}

	for {
		if _, err := jsonScanner.PeekNonWhiteSpaceRune(); err == io.EOF {
			break
		}
		for expectedIndex, expected := range expectedTokens {
			for scannerIndex, scanner := range scanners {
				if scanner.CanProcess() {
					actual, err := scanner.Scan()
					if err != nil {
						t.Logf("scanners[%d]: ", scannerIndex)
						t.Error(err)
					}
					if !expected.IsEqualTo(actual) {
						t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
					}
					break
				}
			}
		}
	}
}
示例#8
0
func TestTokenizerNextToken(t *testing.T) {
	testData := `
	{
		"key": [
			{
				"inner1": "stringValue",
				"inner2": true,
				"inner3": false,
				"inner4": null,
				"inner5": {
					"nested": -2.5e-3
				}
			}
		]
	}
	`
	target := tokenizer.NewTokenizer(strings.NewReader(testData))
	expectedTokens := []token.Token{
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
		token.NewToken(token.KindString, "key"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),

		token.NewToken(token.KindString, "inner1"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindString, "stringValue"),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner2"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindTrue, string(token.KindTrue)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner3"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindFalse, string(token.KindFalse)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner4"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindNull, string(token.KindNull)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner5"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),

		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),

		token.NewToken(token.KindString, "nested"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindNumber, "-2.5e-3"),

		token.NewToken(token.KindEndObject, string(token.KindEndObject)),

		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
		token.NewToken(token.KindEndArray, string(token.KindEndArray)),
		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
	}
	for expectedIndex, expected := range expectedTokens {
		actual, err := target.NextToken()
		if err != nil {
			t.Error(err)
		}
		if !expected.IsEqualTo(actual) {
			t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
		}
	}
	lastToken, err := target.NextToken()
	if err != nil {
		t.Error(err)
	}
	if !token.NewTokenEOF().IsEqualTo(lastToken) {
		t.Error("EOF is expected")
	}
}