// Ensure the scanner can scan a series of tokens correctly. func TestScanner_Scan_Multi(t *testing.T) { type result struct { tok lexer.Token pos lexer.Pos lit string } exp := []result{ {tok: SELECT, pos: lexer.Pos{Line: 0, Char: 0}, lit: ""}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 6}, lit: " "}, {tok: lexer.IDENT, pos: lexer.Pos{Line: 0, Char: 7}, lit: "value"}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 12}, lit: " "}, {tok: FROM, pos: lexer.Pos{Line: 0, Char: 13}, lit: ""}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 17}, lit: " "}, {tok: lexer.IDENT, pos: lexer.Pos{Line: 0, Char: 18}, lit: "myseries"}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 26}, lit: " "}, {tok: WHERE, pos: lexer.Pos{Line: 0, Char: 27}, lit: ""}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 32}, lit: " "}, {tok: lexer.IDENT, pos: lexer.Pos{Line: 0, Char: 33}, lit: "a"}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 34}, lit: " "}, {tok: lexer.EQ, pos: lexer.Pos{Line: 0, Char: 35}, lit: ""}, {tok: lexer.WS, pos: lexer.Pos{Line: 0, Char: 36}, lit: " "}, {tok: lexer.STRING, pos: lexer.Pos{Line: 0, Char: 36}, lit: "b"}, {tok: lexer.EOF, pos: lexer.Pos{Line: 0, Char: 40}, lit: ""}, } // Create a scanner. v := `SELECT value from myseries WHERE a = 'b'` s := lexer.NewScanner(strings.NewReader(v)) // Continually scan until we reach the end. var act []result for { tok, pos, lit := s.Scan() act = append(act, result{tok, pos, lit}) if tok == lexer.EOF { break } } // Verify the token counts match. if len(exp) != len(act) { t.Fatalf("token count mismatch: exp=%d, got=%d", len(exp), len(act)) } // Verify each token matches. for i := range exp { if !reflect.DeepEqual(exp[i], act[i]) { t.Fatalf("%d. token mismatch:\n\nexp=%#v\n\ngot=%#v", i, exp[i], act[i]) } } }
func main() { flag.Parse() l := lexer.NewScanner(os.Stdin) for { tok, pos, lit := l.Scan() // exit if EOF if tok == lexer.EOF { break } // skip whitespace tokens if tok == lexer.WS && *skipWhitespace { continue } // Print token if len(lit) > 0 { fmt.Printf("[%4d:%-3d] %10s - %s\n", pos.Line, pos.Char, tok, strconv.QuoteToASCII(lit)) } else { fmt.Printf("[%4d:%-3d] %10s\n", pos.Line, pos.Char, tok) } } }
// Ensure the scanner can scan tokens correctly. func TestScanner_Scan(t *testing.T) { var tests = []struct { s string tok lexer.Token lit string pos lexer.Pos }{ // Types {s: `STRING`, tok: STRING}, {s: `UINT8`, tok: UINT8}, {s: `INT8`, tok: INT8}, {s: `UINT16`, tok: UINT16}, {s: `INT16`, tok: INT16}, {s: `UINT32`, tok: UINT32}, {s: `INT32`, tok: INT32}, {s: `UINT64`, tok: UINT64}, {s: `INT64`, tok: INT64}, {s: `FLOAT32`, tok: FLOAT32}, {s: `FLOAT64`, tok: FLOAT64}, {s: `TIMESTAMP`, tok: TIMESTAMP}, {s: `BOOLEAN`, tok: BOOLEAN}, // Keywords {s: `ADD`, tok: ADD}, {s: `BY`, tok: BY}, {s: `CLUSTERED`, tok: CLUSTERED}, {s: `CREATE`, tok: CREATE}, {s: `DESCRIBE`, tok: DESCRIBE}, {s: `FOR`, tok: FOR}, {s: `FROM`, tok: FROM}, {s: `INSERT`, tok: INSERT}, {s: `LIMIT`, tok: LIMIT}, {s: `LOG`, tok: LOG}, {s: `NAMESPACE`, tok: NAMESPACE}, {s: `OFFSET`, tok: OFFSET}, {s: `ON`, tok: ON}, {s: `OPTIONAL`, tok: OPTIONAL}, {s: `OPTIONS`, tok: OPTIONS}, {s: `PASSWORD`, tok: PASSWORD}, {s: `PERMISSION`, tok: PERMISSION}, {s: `REMOVE`, tok: REMOVE}, {s: `REQUIRED`, tok: REQUIRED}, {s: `ROLE`, tok: ROLE}, {s: `SELECT`, tok: SELECT}, {s: `SET`, tok: SET}, {s: `SHOW`, tok: SHOW}, {s: `SUBSCRIBE`, tok: SUBSCRIBE}, {s: `TO`, tok: TO}, {s: `TYPE`, tok: TYPE}, {s: `UNSUBSCRIBE`, tok: UNSUBSCRIBE}, {s: `UPDATE`, tok: UPDATE}, {s: `USE`, tok: USE}, {s: `USER`, tok: USER}, {s: `USING`, tok: USING}, {s: `VIEW`, tok: VIEW}, {s: `WHERE`, tok: WHERE}, {s: `WITH`, tok: WITH}, } for i, tt := range tests { s := lexer.NewScanner(strings.NewReader(tt.s)) tok, pos, lit := s.Scan() if tt.tok != tok { t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) } else if tt.lit != lit { t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) } } }
// newBufScanner returns a new buffered scanner for a reader. func newBufScanner(r io.Reader) *bufScanner { return &bufScanner{s: lexer.NewScanner(r)} }