Ejemplo n.º 1
0
// PeekToken returns a token at index from current position.
func (tokenizer *Tokenizer) PeekToken(index int) (token.Token, error) {
	err := tokenizer.queueTokensTo(index)
	if err == io.EOF {
		return token.NewTokenEOF(), nil
	}
	if err != nil {
		return token.NewEmptyToken(), err
	}
	return tokenizer.buffer.peek(index), nil
}
Ejemplo n.º 2
0
func TestTokenizerNextToken(t *testing.T) {
	testData := `
	{
		"key": [
			{
				"inner1": "stringValue",
				"inner2": true,
				"inner3": false,
				"inner4": null,
				"inner5": {
					"nested": -2.5e-3
				}
			}
		]
	}
	`
	target := tokenizer.NewTokenizer(strings.NewReader(testData))
	expectedTokens := []token.Token{
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),
		token.NewToken(token.KindString, "key"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindBeginArray, string(token.KindBeginArray)),
		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),

		token.NewToken(token.KindString, "inner1"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindString, "stringValue"),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner2"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindTrue, string(token.KindTrue)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner3"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindFalse, string(token.KindFalse)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner4"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindNull, string(token.KindNull)),
		token.NewToken(token.KindComma, string(token.KindComma)),

		token.NewToken(token.KindString, "inner5"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),

		token.NewToken(token.KindBeginObject, string(token.KindBeginObject)),

		token.NewToken(token.KindString, "nested"),
		token.NewToken(token.KindSeparator, string(token.KindSeparator)),
		token.NewToken(token.KindNumber, "-2.5e-3"),

		token.NewToken(token.KindEndObject, string(token.KindEndObject)),

		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
		token.NewToken(token.KindEndArray, string(token.KindEndArray)),
		token.NewToken(token.KindEndObject, string(token.KindEndObject)),
	}
	for expectedIndex, expected := range expectedTokens {
		actual, err := target.NextToken()
		if err != nil {
			t.Error(err)
		}
		if !expected.IsEqualTo(actual) {
			t.Errorf("expected[%d] = %#v, actual = %#v\n", expectedIndex, expected, actual)
		}
	}
	lastToken, err := target.NextToken()
	if err != nil {
		t.Error(err)
	}
	if !token.NewTokenEOF().IsEqualTo(lastToken) {
		t.Error("EOF is expected")
	}
}