Example #1
0
File: lexer.go Project: 8l/leaf
func (lx *Lexer) scanToken() *tok.Token {
	if lx.eof {
		panic("no more")
	}

	lx.skipWhites()
	lx.savePos()

	if lx.s.Closed() {
		if lx.insertSemi {
			lx.insertSemi = false
			return lx.token(tt.Semi, ";")
		}
		lx.eof = true

		lx.report(lx.s.Err())

		return lx.eofToken()
	}

	s := lx.s
	r := s.Peek()

	switch {
	case runes.IsLetter(r):
		s.ScanIdent()
		lit := s.Accept()
		t := tt.FromIdent(lit)
		return lx.token(t, lit)
	case runes.IsDigit(r):
		lit, t := lx.scanNumber(false)
		return lx.token(t, lit)
	case r == '\'':
		s.Next()
		lit, e := scanner.ScanChar(lx.s)
		lx.report(e)
		return lx.token(tt.Char, lit)
	case r == '"':
		s.Next()
		lit, e := scanner.ScanString(lx.s)
		lx.report(e)
		return lx.token(tt.String, lit)
	case r == '`':
		s.Next()
		lit, e := scanner.ScanRawString(lx.s)
		lx.report(e)
		return lx.token(tt.String, lit)
	}

	s.Next() // at this time, we will always make some progress

	if r == '.' && runes.IsDigit(s.Peek()) {
		lit, t := lx.scanNumber(true)
		return lx.token(t, lit)
	} else if r == '/' {
		r2 := s.Peek()
		if r2 == '/' || r2 == '*' {
			s, e := scanner.ScanComment(lx.s)
			lx.report(e)
			return lx.token(tt.Comment, s)
		}
	}

	t := lx.scanOperator(r)
	lit := s.Accept()
	if t == tt.Semi {
		lit = ";"
	}

	return lx.token(t, lit)
}
Example #2
0
File: lexer.go Project: 8l/leaf
func (lx *Lexer) scanToken() *tok.Token {
	if lx.eof {
		panic("no more")
	}

	lx.skipWhites()
	lx.savePos()

	if lx.s.Closed() {
		if lx.insertSemi {
			lx.insertSemi = false
			return lx.token(tt.Semi, ";")
		}
		lx.eof = true
		lx.report(lx.s.Err())

		return lx.eofToken()
	}

	s := lx.s
	r := s.Peek()

	switch {
	case runes.IsLetter(r):
		s.ScanIdent()
		lit := s.Accept()
		t := tt.FromIdent(lit)
		return lx.token(t, lit)
	case r == '-' || r == '+':
		s.Next() // eat the sign
		if s.Scan('.') {
			lit, t := lx.scanNumber(true)
			return lx.token(t, lit)
		} else if runes.IsDigit(s.Peek()) {
			lit, t := lx.scanNumber(false)
			return lx.token(t, lit)
		} else {
			lx.reportf("a sign must lead a number")
			lit := s.Accept()
			return lx.token(tt.Illegal, lit)
		}
	case runes.IsDigit(r):
		lit, t := lx.scanNumber(false)
		return lx.token(t, lit)
	case r == '\'': // start of a char
		s.Next()
		lit, e := scanner.ScanChar(lx.s)
		lx.report(e)
		return lx.token(tt.Char, lit)
	case r == '"': // start of a string
		s.Next()
		lit, e := scanner.ScanString(lx.s)
		lx.report(e)
		return lx.token(tt.String, lit)
	case r == '`': // start of a raw string
		s.Next()
		lit, e := scanner.ScanRawString(lx.s)
		lx.report(e)
		return lx.token(tt.String, lit)
	}

	s.Next() // always make some progress here

	if r == '.' && runes.IsDigit(s.Peek()) {
		lit, t := lx.scanNumber(true)
		return lx.token(t, lit)
	}

	if r == '/' { // start of comment
		next := s.Peek()
		if next == '/' || next == '*' {
			s, e := scanner.ScanComment(lx.s)
			lx.report(e)
			return lx.token(tt.Comment, s)
		}
	}

	t := lx.scanOperator(r)
	lit := s.Accept()
	if t == tt.Semi {
		lit = ";"
	}

	return lx.token(t, lit)
}