func (lx *Lexer) scanToken() *tok.Token {
if lx.eof {
panic("no more")
}
lx.skipWhites()
lx.savePos()
if lx.s.Closed() {
if lx.insertSemi {
lx.insertSemi = false
return lx.token(tt.Semi, ";")
}
lx.eof = true
lx.report(lx.s.Err())
return lx.eofToken()
}
s := lx.s
r := s.Peek()
switch {
case runes.IsLetter(r):
s.ScanIdent()
lit := s.Accept()
t := tt.FromIdent(lit)
return lx.token(t, lit)
case runes.IsDigit(r):
lit, t := lx.scanNumber(false)
return lx.token(t, lit)
case r == '\'':
s.Next()
lit, e := scanner.ScanChar(lx.s)
lx.report(e)
return lx.token(tt.Char, lit)
case r == '"':
s.Next()
lit, e := scanner.ScanString(lx.s)
lx.report(e)
return lx.token(tt.String, lit)
case r == '`':
s.Next()
lit, e := scanner.ScanRawString(lx.s)
lx.report(e)
return lx.token(tt.String, lit)
}
s.Next() // at this time, we will always make some progress
if r == '.' && runes.IsDigit(s.Peek()) {
lit, t := lx.scanNumber(true)
return lx.token(t, lit)
} else if r == '/' {
r2 := s.Peek()
if r2 == '/' || r2 == '*' {
s, e := scanner.ScanComment(lx.s)
lx.report(e)
return lx.token(tt.Comment, s)
}
}
t := lx.scanOperator(r)
lit := s.Accept()
if t == tt.Semi {
lit = ";"
}
return lx.token(t, lit)
}
func (lx *Lexer) scanToken() *tok.Token {
if lx.eof {
panic("no more")
}
lx.skipWhites()
lx.savePos()
if lx.s.Closed() {
if lx.insertSemi {
lx.insertSemi = false
return lx.token(tt.Semi, ";")
}
lx.eof = true
lx.report(lx.s.Err())
return lx.eofToken()
}
s := lx.s
r := s.Peek()
switch {
case runes.IsLetter(r):
s.ScanIdent()
lit := s.Accept()
t := tt.FromIdent(lit)
return lx.token(t, lit)
case r == '-' || r == '+':
s.Next() // eat the sign
if s.Scan('.') {
lit, t := lx.scanNumber(true)
return lx.token(t, lit)
} else if runes.IsDigit(s.Peek()) {
lit, t := lx.scanNumber(false)
return lx.token(t, lit)
} else {
lx.reportf("a sign must lead a number")
lit := s.Accept()
return lx.token(tt.Illegal, lit)
}
case runes.IsDigit(r):
lit, t := lx.scanNumber(false)
return lx.token(t, lit)
case r == '\'': // start of a char
s.Next()
lit, e := scanner.ScanChar(lx.s)
lx.report(e)
return lx.token(tt.Char, lit)
case r == '"': // start of a string
s.Next()
lit, e := scanner.ScanString(lx.s)
lx.report(e)
return lx.token(tt.String, lit)
case r == '`': // start of a raw string
s.Next()
lit, e := scanner.ScanRawString(lx.s)
lx.report(e)
return lx.token(tt.String, lit)
}
s.Next() // always make some progress here
if r == '.' && runes.IsDigit(s.Peek()) {
lit, t := lx.scanNumber(true)
return lx.token(t, lit)
}
if r == '/' { // start of comment
next := s.Peek()
if next == '/' || next == '*' {
s, e := scanner.ScanComment(lx.s)
lx.report(e)
return lx.token(tt.Comment, s)
}
}
t := lx.scanOperator(r)
lit := s.Accept()
if t == tt.Semi {
lit = ";"
}
return lx.token(t, lit)
}