Exemple #1
0
func TestLexer_new(t *testing.T) {

	l := lexer.New(bytes.NewBufferString("a 'b `test` 123 1.23 test {}"))

	var cnt = 0
	for tkn := l.NextToken(); tkn.GetType() != token.TEOF; tkn = l.NextToken() {

		if tkn.GetType() == token.TSpace {
			continue
		}
		kit.Printf("token type: %v token value: %s", tkn.GetType(), tkn.GetVal())

		switch cnt {
		case 0:
			assertTokenType(t, tkn.GetType(), token.TIdentifier)
		case 1:
			assertTokenType(t, tkn.GetType(), token.TIdentifier)
		case 2:
			assertTokenType(t, tkn.GetType(), token.TString)
		case 3:
			assertTokenType(t, tkn.GetType(), token.TNumber)
		case 4:
			assertTokenType(t, tkn.GetType(), token.TNumber)
		case 5:
			assertTokenType(t, tkn.GetType(), token.TIdentifier)
		case 6:
			assertTokenType(t, tkn.GetType(), token.TLeftBrace)
		case 7:
			assertTokenType(t, tkn.GetType(), token.TRightBrace)
		}

		cnt++
	}
}
Exemple #2
0
Fichier : lexer.go Projet : mk2/yon
func (l *lexer) ReadToken() (kit.Token, error) {

	if l.onceAgainToken {

		kit.Printf("found unused last token: %+v\n", l.lastToken)

		l.Lock()
		l.onceAgainToken = false
		l.Unlock()

		if l.lastToken == nil {
			return nil, errors.New("no last read token")
		}

		return l.lastToken, nil
	}

	kit.Println("waiting for incoming token")

	select {

	case t := <-l.tokens:
		l.lastToken = t
		return t, nil

	case <-time.After(kit.LexerTimeout):
		// timeout

	}

	return nil, errors.New("no token gained")
}
Exemple #3
0
Fichier : lexer.go Projet : mk2/yon
func lex(l *lexer) stateFn {

	switch r := l.peek(); {

	case r == '(':
		l.emit(token.TLeftParen)
		l.next()

	case r == ')':
		l.emit(token.TRightParen)
		l.next()

	case r == '{':
		l.emit(token.TLeftBrace)
		l.next()

	case r == '}':
		l.emit(token.TRightBrace)
		l.next()

	case r == '[':
		l.emit(token.TLeftSquareBracket)
		l.next()

	case r == ']':
		l.emit(token.TRightSquareBracket)
		l.next()

	case r == ':':
		l.emit(token.TDblColon)
		l.next()

	case r == ';':
		l.emit(token.TSemiColon)
		l.next()

	case r == '"' || r == '`':
		l.leftDelim = r
		return lexString

	case isNumber(r):
		return lexNumber

	case isSpace(r):
		return lexSpace

	case isLetter(r):
		return lexIdentifier

	case r == nilRune:
		l.emit(token.TEOF)
		return nil

	default:
		kit.Printf("no matching: %v", r)

	}

	return lex
}
Exemple #4
0
func (ip *interp) run(words kit.WordScanner) {

	m := ip.memo

	var (
		w   kit.Word
		err error
	)

	kit.Println("start RUN_LOOP")

RUN_LOOP:
	for {

		if w, err = words.ReadWord(); err != nil {
			ip.errorCh <- err
			break RUN_LOOP
		}

		kit.Printf("word: %+v", w)

		switch w.GetWordType() {

		case word.TNumberWord:
			kit.Println("number word")
			if _, err := w.Do(m); err != nil {
				ip.errorCh <- err
				break
			}

		case word.TStringWord:
			kit.Println("string word")
			if _, err := w.Do(m); err != nil {
				ip.errorCh <- err
				break
			}

		case word.TBoolWord:
			kit.Println("bool word")
			if _, err := w.Do(m); err != nil {
				ip.errorCh <- err
				break
			}

		case word.TArrayWord:
			kit.Println("array word")
			if _, err := w.Do(m); err != nil {
				ip.errorCh <- err
				break
			}

		case word.TFuncWord:
			kit.Println("func word")
			if _, err := w.Do(m); err != nil {
				ip.errorCh <- err
				break
			}

		case word.TNilWord:
			kit.Println("nil word")
			break RUN_LOOP

		default:
			kit.Printf("unknown word: %+v\n", w)
			ip.errorCh <- errors.New("unknown word")
			break RUN_LOOP

		}

	}

	kit.Println("exit RUN_LOOP")

	ip.stoppedCh <- struct{}{}
}