Пример #1
0
// Ensure the scanner can scan a series of tokens correctly.
func TestScanner_Scan_Multi(t *testing.T) {
	type result struct {
		tok lexer.Token
		pos lexer.Pos
		lit string
	}
	exp := []result{
		{tok: SELECT, pos: lexer.Pos{Line: 0, Char: 0}, lit: ""},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 6}, lit: " "},
		{tok: lexer.IDENT, pos: lexer.Pos{Line: 0, Char: 7}, lit: "value"},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 12}, lit: " "},
		{tok: FROM, pos: lexer.Pos{Line: 0, Char: 13}, lit: ""},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 17}, lit: " "},
		{tok: lexer.IDENT, pos: lexer.Pos{Line: 0, Char: 18}, lit: "myseries"},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 26}, lit: " "},
		{tok: WHERE, pos: lexer.Pos{Line: 0, Char: 27}, lit: ""},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 32}, lit: " "},
		{tok: lexer.IDENT, pos: lexer.Pos{Line: 0, Char: 33}, lit: "a"},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 34}, lit: " "},
		{tok: lexer.EQ, pos: lexer.Pos{Line: 0, Char: 35}, lit: ""},
		{tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 36}, lit: " "},
		{tok: lexer.STRING, pos: lexer.Pos{Line: 0, Char: 36}, lit: "b"},
		{tok: lexer.EOF, pos: lexer.Pos{Line: 0, Char: 40}, lit: ""},
	}

	// Create a scanner.
	v := `SELECT value from myseries WHERE a = 'b'`
	s := lexer.NewScanner(strings.NewReader(v))

	// Continually scan until we reach the end.
	var act []result
	for {
		tok, pos, lit := s.Scan()
		act = append(act, result{tok, pos, lit})
		if tok == lexer.EOF {
			break
		}
	}

	// Verify the token counts match.
	if len(exp) != len(act) {
		t.Fatalf("token count mismatch: exp=%d, got=%d", len(exp), len(act))
	}

	// Verify each token matches.
	for i := range exp {
		if !reflect.DeepEqual(exp[i], act[i]) {
			t.Fatalf("%d. token mismatch:\n\nexp=%#v\n\ngot=%#v", i, exp[i], act[i])
		}
	}
}
Пример #2
0
func main() {
	flag.Parse()
	l := lexer.NewScanner(os.Stdin)
	for {
		tok, pos, lit := l.Scan()

		// exit if EOF
		if tok == lexer.EOF {
			break
		}

		// skip whitespace tokens
		if tok == lexer.WS && *skipWhitespace {
			continue
		}

		// Print token
		if len(lit) > 0 {
			fmt.Printf("[%4d:%-3d] %10s - %s\n", pos.Line, pos.Char, tok, strconv.QuoteToASCII(lit))
		} else {
			fmt.Printf("[%4d:%-3d] %10s\n", pos.Line, pos.Char, tok)
		}
	}
}
Пример #3
0
// Ensure the scanner can scan tokens correctly.
func TestScanner_Scan(t *testing.T) {
	var tests = []struct {
		s   string
		tok lexer.Token
		lit string
		pos lexer.Pos
	}{

		// Types
		{s: `STRING`, tok: STRING},
		{s: `UINT8`, tok: UINT8},
		{s: `INT8`, tok: INT8},
		{s: `UINT16`, tok: UINT16},
		{s: `INT16`, tok: INT16},
		{s: `UINT32`, tok: UINT32},
		{s: `INT32`, tok: INT32},
		{s: `UINT64`, tok: UINT64},
		{s: `INT64`, tok: INT64},
		{s: `FLOAT32`, tok: FLOAT32},
		{s: `FLOAT64`, tok: FLOAT64},
		{s: `TIMESTAMP`, tok: TIMESTAMP},
		{s: `BOOLEAN`, tok: BOOLEAN},

		// Keywords
		{s: `ADD`, tok: ADD},
		{s: `BY`, tok: BY},
		{s: `CLUSTERED`, tok: CLUSTERED},
		{s: `CREATE`, tok: CREATE},
		{s: `DESCRIBE`, tok: DESCRIBE},
		{s: `FOR`, tok: FOR},
		{s: `FROM`, tok: FROM},
		{s: `INSERT`, tok: INSERT},
		{s: `LIMIT`, tok: LIMIT},
		{s: `LOG`, tok: LOG},
		{s: `NAMESPACE`, tok: NAMESPACE},
		{s: `OFFSET`, tok: OFFSET},
		{s: `ON`, tok: ON},
		{s: `OPTIONAL`, tok: OPTIONAL},
		{s: `OPTIONS`, tok: OPTIONS},
		{s: `PASSWORD`, tok: PASSWORD},
		{s: `PERMISSION`, tok: PERMISSION},
		{s: `REMOVE`, tok: REMOVE},
		{s: `REQUIRED`, tok: REQUIRED},
		{s: `ROLE`, tok: ROLE},
		{s: `SELECT`, tok: SELECT},
		{s: `SET`, tok: SET},
		{s: `SHOW`, tok: SHOW},
		{s: `SUBSCRIBE`, tok: SUBSCRIBE},
		{s: `TO`, tok: TO},
		{s: `TYPE`, tok: TYPE},
		{s: `UNSUBSCRIBE`, tok: UNSUBSCRIBE},
		{s: `UPDATE`, tok: UPDATE},
		{s: `USE`, tok: USE},
		{s: `USER`, tok: USER},
		{s: `USING`, tok: USING},
		{s: `VIEW`, tok: VIEW},
		{s: `WHERE`, tok: WHERE},
		{s: `WITH`, tok: WITH},
	}

	for i, tt := range tests {
		s := lexer.NewScanner(strings.NewReader(tt.s))
		tok, pos, lit := s.Scan()
		if tt.tok != tok {
			t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
		} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
			t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
		} else if tt.lit != lit {
			t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
		}
	}
}
Пример #4
0
// newBufScanner returns a new buffered scanner for a reader.
func newBufScanner(r io.Reader) *bufScanner {
	return &bufScanner{s: lexer.NewScanner(r)}
}