Example #1
0
func TestLexlLexer(t *testing.T) {
	lexl0Grammar := GenerateLexl0Grammar()
	lexlL0sr := GenerateLexl0SR()
	for _, block := range lexlL0sr {
		fmt.Println(MatchBlockToString(block))
	}
	ndfa, err := lexlL0sr.ConstructNdfa()
	if err != nil {
		t.Error(err)
		return
	}
	fmt.Println("NDFA:")
	fmt.Println(NdfaToString(ndfa))
	dfa, err := ndfa.TransformToDfa()
	if err != nil {
		t.Error(err)
		return
	}
	fmt.Println("DFA:")
	fmt.Println(DfaToString(dfa))
	lexer, err := GenerateLexlLexerFromDfa(dfa, lexl0Grammar)
	if err != nil {
		t.Error(err)
		return
	}
	in := parser.NewStringReader(lexl0Text)
	lex, err := lexer.Open(in)
	if err != nil {
		t.Error(err)
		return
	}
	for {
		hasMore, err := lex.HasMoreTokens()
		if err != nil {
			t.Error(err)
			return
		}
		if !hasMore {
			break
		}
		tok, err := lex.NextToken()
		if err != nil {
			t.Error(err)
			return
		}
		fmt.Println(">" + tok.Terminal().Name())
	}
	fmt.Println("finished")
}
Example #2
0
func TestParser(t *testing.T) {
	gb := parser.OpenGrammarBuilder()

	gb.Terminals("NONTERM", "COLEQ", "PIPE", "IDENTIFIER").
		Nonterminals("bnf", "ntdecl", "def", "ntort").
		Rule().Lhs("bnf").Rhs("ntdecl").
		Rule().Lhs("bnf").Rhs("ntdecl", "bnf").
		Rule().Lhs("ntdecl").Rhs("NONTERM", "COLEQ", "def").
		Rule().Lhs("ntdecl").Rhs("ntdecl", "PIPE", "def").
		Rule().Lhs("def").Rhs("ntort").
		Rule().Lhs("def").Rhs("ntort", "def").
		Rule().Lhs("ntort").Rhs("IDENTIFIER").
		Rule().Lhs("ntort").Rhs("NONTERM").
		Rule().Lhs("`*").Rhs("bnf", "`.").
		Name("simple-bnf")

	g, err := gb.Build()
	if err != nil {
		t.Error(err)
	}

	metaBnf := `
	
		<bnf> := <ntdecl>
		      |  <ntdecl> <bnf>
		
		<ntdecl> := NONTERM COLEQ <def>
		         |  <ntdecl> PIPE <def>
				
		<def> := <ntort>
			  |  <ntort> <def>
		
		<ntort> := IDENTIFIER
		        |  NONTERM
				
	`
	lexer, err := NewSimpleBnfLexer(g)
	if err != nil {
		t.Error(err)
		return
	}

	lexer.Reset(parser.NewStringReader(metaBnf))
	p, err := GenerateParser(g)
	if err != nil {
		t.Error(err)
		return
	}

	var ast parser.SyntaxTreeNode
	ast, err = p.Parse(lexer, nil)
	if err != nil {
		t.Error(err)
		return
	}

	fmt.Println("Got AST.")
	fmt.Printf("Top is: %s\n", ast.Part())

	cmap := make(map[parser.SyntaxTreeNode]int)
	outmap := make(map[int]string)
	var cnid, chid int

	DumpTree(ast.(*astNode))

	stack := []parser.SyntaxTreeNode{ast}
	for len(stack) > 0 {
		var buf []byte
		cn := stack[len(stack)-1]
		stack = stack[0 : len(stack)-1]
		if id, has := cmap[cn]; !has {
			cnid = len(cmap)
			cmap[cn] = cnid
		} else {
			cnid = id
		}
		var desc string
		if cn.Part() == nil {
			desc = "-"
		} else {
			desc = cn.Part().String()
		}
		var valstr string
		if cn.NumChildren() == 0 {
			if vt, isVal := cn.Part().(*parser.ValueTerminal); isVal {
				val := vt.Value()
				if s, isStr := val.(string); isStr {
					valstr = fmt.Sprintf("\"%s\"", s)
				} else if s, isStr := val.(parser.Stringable); isStr {
					valstr = fmt.Sprintf("\"%s\"", s.String())
				} else {
					valstr = fmt.Sprintf("0x%8.8X", reflect.ValueOf(val).Pointer())
				}
			} else {
				valstr = "-"
			}
			var teststr string
			if cn.(*astNode).left == nil {
				teststr = "."
			} else {
				teststr = fmt.Sprintf("%d", cn.(*astNode).left.first)
			}
			buf = append(buf, fmt.Sprintf("[%d: leaf %s %s %s (%d-%d)]", cnid, teststr, desc, valstr, cn.First(), cn.Last())...)
		} else {
			buf = append(buf, fmt.Sprintf("[%d: %s {", cnid, cn.Rule().String())...)
			for i := 0; i < cn.NumChildren(); i++ {
				child := cn.Child(i)
				stack = append(stack, child)
				if id, has := cmap[child]; !has {
					chid = len(cmap)
					cmap[child] = chid
				} else {
					chid = id
				}
				buf = append(buf, fmt.Sprintf("%d", chid)...)
				if i < cn.NumChildren()-1 {
					buf = append(buf, ","...)
				}
			}
			buf = append(buf, fmt.Sprintf("} (%d-%d)]", cn.First(), cn.Last())...)
		}
		outmap[cnid] = string(buf)
	}
	for i := 0; i < len(outmap); i++ {
		fmt.Println(outmap[i])
	}
}
Example #3
0
func TestRelex(t *testing.T) {

	gb := parser.OpenGrammarBuilder()
	gb.Name("simple-calculator").
		Terminals("ID", "EQ", "PRINT", "POW", "PLUS", "MINUS", "TIMES", "DIV", "MOD", "LP", "RP", "NUM").
		Nonterminals("program", "statement", "assignment", "output", "expr", "aopfree", "aop",
			"mopfree", "mop", "unit").
		Rule().Lhs("`*").Rhs("program", "`.").
		Rule().Lhs("program").Rhs("statement").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return &StatementList{statements: []Statement{values[0].(Statement)}}, nil
		}).
		Rule().Lhs("program").Rhs("statement", "program").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			slist := values[1].(*StatementList)
			slist.statements = append(slist.statements, values[0].(Statement))
			return slist, nil
		}).
		Rule().Lhs("statement").Rhs("assignment").
		Rule().Lhs("statement").Rhs("output").
		Rule().Lhs("assignment").Rhs("ID", "EQ", "expr").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return &Assignment{varname: values[0].(parser.Stringable).String(),
				value: values[2].(Expression)}, nil
		}).
		Rule().Lhs("output").Rhs("PRINT", "expr").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return &Output{value: values[1].(Expression)}, nil
		}).
		Rule().Lhs("expr").Rhs("aopfree").
		Rule().Lhs("expr").Rhs("expr", "aop", "aopfree").Value(binaryOpCtor).
		Rule().Lhs("aopfree").Rhs("aopfree", "mop", "mopfree").Value(binaryOpCtor).
		Rule().Lhs("mopfree").Rhs("mopfree", "POW", "unit").Value(binaryOpCtor).
		Rule().Lhs("aop").Rhs("PLUS").
		Rule().Lhs("aop").Rhs("MINUS").
		Rule().Lhs("mop").Rhs("TIMES").
		Rule().Lhs("mop").Rhs("DIV").
		Rule().Lhs("mop").Rhs("MOD").
		Rule().Lhs("unit").Rhs("ID").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return &VariableExpression{varname: values[0].(parser.Stringable).String()}, nil
		}).
		Rule().Lhs("unit").Rhs("MINUS", "unit").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return &Negation{arg: values[1].(Expression)}, nil
		}).
		Rule().Lhs("unit").Rhs("LP", "expr", "RP").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return values[1], nil
		}).
		Rule().Lhs("unit").Rhs("NUM").
		Value(func(p parser.Production, values []interface{}) (interface{}, error) {
			return &Literal{val: values[0].(*big.Int)}, nil
		})
	g, err := gb.Build()
	if err != nil {
		t.Error(err)
		return
	}

	p, err := earley.GenerateParser(g)
	if err != nil {
		t.Error("parser generation failed: " + err.Error())
		return
	}
	fmt.Println(g.Name())

	lb := OpenLexerBuilder(g)
	lb.Token("ID").Expr(`([a-zA-Z][a-zA-Z0-9]*)`)
	lb.Token("EQ").Expr(`=`)
	lb.Token("PRINT").Expr(`print`)
	lb.Token("POW").Expr(`^`).
		Value(func(part parser.GrammarParticle, match string) interface{} {
			return EXPONENTIATION
		})
	lb.Token("PLUS").Expr(`\+`).
		Value(func(part parser.GrammarParticle, match string) interface{} {
			return ADDITION
		})
	lb.Token("MINUS").Expr(`-`).
		Value(func(part parser.GrammarParticle, match string) interface{} {
			return SUBTRACTION
		})
	lb.Token("TIMES").Expr(`\*`).
		Value(func(part parser.GrammarParticle, match string) interface{} {
			return MULTIPLICATION
		})
	lb.Token("DIV").Expr(`/`).
		Value(func(part parser.GrammarParticle, match string) interface{} {
			return DIVISION
		})
	lb.Token("MOD").Expr(`%`).
		Value(func(part parser.GrammarParticle, match string) interface{} {
			return MODULUS
		})
	lb.Token("LP").Expr(`\(`)
	lb.Token("RP").Expr(`\)`)
	lb.Token("NUM").Expr("(0|[1-9][0-9]*)").
		Value(func(part parser.GrammarParticle, match string) interface{} {
			n := big.NewInt(0)
			ten := big.NewInt(10)
			for i := len(match) - 1; i >= 0; i-- {
				n.Mul(n, ten).Add(n, big.NewInt(int64(match[i]-'0')))
			}
			return n
		})
	lexer, err := lb.Build()
	if err != nil {
		t.Error("lexer build failed: " + err.Error())
		return
	}

	lexer.Reset(parser.NewStringReader(SAMPLE_PROGRAM))
	ast, err := p.Parse(lexer, nil)
	if err != nil {
		t.Error(err)
		return
	}
	ast = ast
}