func TestLexlLexer(t *testing.T) {
lexl0Grammar := GenerateLexl0Grammar()
lexlL0sr := GenerateLexl0SR()
for _, block := range lexlL0sr {
fmt.Println(MatchBlockToString(block))
}
ndfa, err := lexlL0sr.ConstructNdfa()
if err != nil {
t.Error(err)
return
}
fmt.Println("NDFA:")
fmt.Println(NdfaToString(ndfa))
dfa, err := ndfa.TransformToDfa()
if err != nil {
t.Error(err)
return
}
fmt.Println("DFA:")
fmt.Println(DfaToString(dfa))
lexer, err := GenerateLexlLexerFromDfa(dfa, lexl0Grammar)
if err != nil {
t.Error(err)
return
}
in := parser.NewStringReader(lexl0Text)
lex, err := lexer.Open(in)
if err != nil {
t.Error(err)
return
}
for {
hasMore, err := lex.HasMoreTokens()
if err != nil {
t.Error(err)
return
}
if !hasMore {
break
}
tok, err := lex.NextToken()
if err != nil {
t.Error(err)
return
}
fmt.Println(">" + tok.Terminal().Name())
}
fmt.Println("finished")
}
func TestParser(t *testing.T) {
gb := parser.OpenGrammarBuilder()
gb.Terminals("NONTERM", "COLEQ", "PIPE", "IDENTIFIER").
Nonterminals("bnf", "ntdecl", "def", "ntort").
Rule().Lhs("bnf").Rhs("ntdecl").
Rule().Lhs("bnf").Rhs("ntdecl", "bnf").
Rule().Lhs("ntdecl").Rhs("NONTERM", "COLEQ", "def").
Rule().Lhs("ntdecl").Rhs("ntdecl", "PIPE", "def").
Rule().Lhs("def").Rhs("ntort").
Rule().Lhs("def").Rhs("ntort", "def").
Rule().Lhs("ntort").Rhs("IDENTIFIER").
Rule().Lhs("ntort").Rhs("NONTERM").
Rule().Lhs("`*").Rhs("bnf", "`.").
Name("simple-bnf")
g, err := gb.Build()
if err != nil {
t.Error(err)
}
metaBnf := `
<bnf> := <ntdecl>
| <ntdecl> <bnf>
<ntdecl> := NONTERM COLEQ <def>
| <ntdecl> PIPE <def>
<def> := <ntort>
| <ntort> <def>
<ntort> := IDENTIFIER
| NONTERM
`
lexer, err := NewSimpleBnfLexer(g)
if err != nil {
t.Error(err)
return
}
lexer.Reset(parser.NewStringReader(metaBnf))
p, err := GenerateParser(g)
if err != nil {
t.Error(err)
return
}
var ast parser.SyntaxTreeNode
ast, err = p.Parse(lexer, nil)
if err != nil {
t.Error(err)
return
}
fmt.Println("Got AST.")
fmt.Printf("Top is: %s\n", ast.Part())
cmap := make(map[parser.SyntaxTreeNode]int)
outmap := make(map[int]string)
var cnid, chid int
DumpTree(ast.(*astNode))
stack := []parser.SyntaxTreeNode{ast}
for len(stack) > 0 {
var buf []byte
cn := stack[len(stack)-1]
stack = stack[0 : len(stack)-1]
if id, has := cmap[cn]; !has {
cnid = len(cmap)
cmap[cn] = cnid
} else {
cnid = id
}
var desc string
if cn.Part() == nil {
desc = "-"
} else {
desc = cn.Part().String()
}
var valstr string
if cn.NumChildren() == 0 {
if vt, isVal := cn.Part().(*parser.ValueTerminal); isVal {
val := vt.Value()
if s, isStr := val.(string); isStr {
valstr = fmt.Sprintf("\"%s\"", s)
} else if s, isStr := val.(parser.Stringable); isStr {
valstr = fmt.Sprintf("\"%s\"", s.String())
} else {
valstr = fmt.Sprintf("0x%8.8X", reflect.ValueOf(val).Pointer())
}
} else {
valstr = "-"
}
var teststr string
if cn.(*astNode).left == nil {
teststr = "."
} else {
teststr = fmt.Sprintf("%d", cn.(*astNode).left.first)
}
buf = append(buf, fmt.Sprintf("[%d: leaf %s %s %s (%d-%d)]", cnid, teststr, desc, valstr, cn.First(), cn.Last())...)
} else {
buf = append(buf, fmt.Sprintf("[%d: %s {", cnid, cn.Rule().String())...)
for i := 0; i < cn.NumChildren(); i++ {
child := cn.Child(i)
stack = append(stack, child)
if id, has := cmap[child]; !has {
chid = len(cmap)
cmap[child] = chid
} else {
chid = id
}
buf = append(buf, fmt.Sprintf("%d", chid)...)
if i < cn.NumChildren()-1 {
buf = append(buf, ","...)
}
}
buf = append(buf, fmt.Sprintf("} (%d-%d)]", cn.First(), cn.Last())...)
}
outmap[cnid] = string(buf)
}
for i := 0; i < len(outmap); i++ {
fmt.Println(outmap[i])
}
}
func TestRelex(t *testing.T) {
gb := parser.OpenGrammarBuilder()
gb.Name("simple-calculator").
Terminals("ID", "EQ", "PRINT", "POW", "PLUS", "MINUS", "TIMES", "DIV", "MOD", "LP", "RP", "NUM").
Nonterminals("program", "statement", "assignment", "output", "expr", "aopfree", "aop",
"mopfree", "mop", "unit").
Rule().Lhs("`*").Rhs("program", "`.").
Rule().Lhs("program").Rhs("statement").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return &StatementList{statements: []Statement{values[0].(Statement)}}, nil
}).
Rule().Lhs("program").Rhs("statement", "program").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
slist := values[1].(*StatementList)
slist.statements = append(slist.statements, values[0].(Statement))
return slist, nil
}).
Rule().Lhs("statement").Rhs("assignment").
Rule().Lhs("statement").Rhs("output").
Rule().Lhs("assignment").Rhs("ID", "EQ", "expr").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return &Assignment{varname: values[0].(parser.Stringable).String(),
value: values[2].(Expression)}, nil
}).
Rule().Lhs("output").Rhs("PRINT", "expr").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return &Output{value: values[1].(Expression)}, nil
}).
Rule().Lhs("expr").Rhs("aopfree").
Rule().Lhs("expr").Rhs("expr", "aop", "aopfree").Value(binaryOpCtor).
Rule().Lhs("aopfree").Rhs("aopfree", "mop", "mopfree").Value(binaryOpCtor).
Rule().Lhs("mopfree").Rhs("mopfree", "POW", "unit").Value(binaryOpCtor).
Rule().Lhs("aop").Rhs("PLUS").
Rule().Lhs("aop").Rhs("MINUS").
Rule().Lhs("mop").Rhs("TIMES").
Rule().Lhs("mop").Rhs("DIV").
Rule().Lhs("mop").Rhs("MOD").
Rule().Lhs("unit").Rhs("ID").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return &VariableExpression{varname: values[0].(parser.Stringable).String()}, nil
}).
Rule().Lhs("unit").Rhs("MINUS", "unit").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return &Negation{arg: values[1].(Expression)}, nil
}).
Rule().Lhs("unit").Rhs("LP", "expr", "RP").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return values[1], nil
}).
Rule().Lhs("unit").Rhs("NUM").
Value(func(p parser.Production, values []interface{}) (interface{}, error) {
return &Literal{val: values[0].(*big.Int)}, nil
})
g, err := gb.Build()
if err != nil {
t.Error(err)
return
}
p, err := earley.GenerateParser(g)
if err != nil {
t.Error("parser generation failed: " + err.Error())
return
}
fmt.Println(g.Name())
lb := OpenLexerBuilder(g)
lb.Token("ID").Expr(`([a-zA-Z][a-zA-Z0-9]*)`)
lb.Token("EQ").Expr(`=`)
lb.Token("PRINT").Expr(`print`)
lb.Token("POW").Expr(`^`).
Value(func(part parser.GrammarParticle, match string) interface{} {
return EXPONENTIATION
})
lb.Token("PLUS").Expr(`\+`).
Value(func(part parser.GrammarParticle, match string) interface{} {
return ADDITION
})
lb.Token("MINUS").Expr(`-`).
Value(func(part parser.GrammarParticle, match string) interface{} {
return SUBTRACTION
})
lb.Token("TIMES").Expr(`\*`).
Value(func(part parser.GrammarParticle, match string) interface{} {
return MULTIPLICATION
})
lb.Token("DIV").Expr(`/`).
Value(func(part parser.GrammarParticle, match string) interface{} {
return DIVISION
})
lb.Token("MOD").Expr(`%`).
Value(func(part parser.GrammarParticle, match string) interface{} {
return MODULUS
})
lb.Token("LP").Expr(`\(`)
lb.Token("RP").Expr(`\)`)
lb.Token("NUM").Expr("(0|[1-9][0-9]*)").
Value(func(part parser.GrammarParticle, match string) interface{} {
n := big.NewInt(0)
ten := big.NewInt(10)
for i := len(match) - 1; i >= 0; i-- {
n.Mul(n, ten).Add(n, big.NewInt(int64(match[i]-'0')))
}
return n
})
lexer, err := lb.Build()
if err != nil {
t.Error("lexer build failed: " + err.Error())
return
}
lexer.Reset(parser.NewStringReader(SAMPLE_PROGRAM))
ast, err := p.Parse(lexer, nil)
if err != nil {
t.Error(err)
return
}
ast = ast
}