main.go

// Copyright 2015 The YY Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command yy processes yacc source code and produces three output files:
//
// - A Go file containing definitions of AST nodes.
//
// - A Go file containing documentation examples[0] of productions defined by the yacc grammar.
//
// - A new yacc file with automatic actions instantiating the AST nodes.
//
// Installation
//
// To install yy
//
//     $ go get [-u] github.com/cznic/yy
//
// Online documentation
//
// http://godoc.org/github.com/cznic/yy
//
// Usage
//
// Invocation:
//
//     $ yy [options] <input.y>
//
// Options
//
// Flags handled by the yy command:
//
//     -ast string
//           Output AST nodes definitions. (default "ast.go")
//     -astExamples string
//           Output AST examples. (default "ast_test.go")
//     -astImport string
//           Optional AST file imports.
//     -exampleAST string
//           Fuction to call to produce example ASTs. (default "exampleAST")
//     -kind string
//           Default node kind (rule case) field name. (default "kind")
//     -namedCases
//           Generate typed and named case numbers.
//     -node string
//           Default non terminal yacc type. (default "node")
//     -o string
//           Output yacc file. (default "parser.y")
//     -pkg string
//           Package name of generated Go files. Extract from input when blank.
//     -prettyString string
//           Fuction to stringify things nicely. (default "prettyString")
//     -token string
//           Default terminal yacc type. (default "Token")
//     -tokenSep string
//           AST examples token separator string. (default " ")
//     -v string
//           create grammar report (default "y.output")
//     -yylex string
//           Type of yacc's yylex. (default "*lexer")
//
// Changelog
//
// 2017-10-23: Added the case directive.
//
// Examples
//
// A partial example: see the testdata directory and files
//
//     input:	in.y
//     output:	ast.go
//     output:	ast_test.go
//     output:	out.y
//
// The three output files were generated by
//
//     yy -o testdata/out.y -ast testdata/ast.go -astExamples testdata/ast_test.go testdata/in.y
//
// A more complete, working project using yy can be found at http://godoc.org/github.com/cznic/pl0
//
// Concepts
//
// Every rule is turned into a definition of a struct type in ast.go (adjust using the -ast flag). The fields of the type are a sum of all productions (cases) of the rule.
//
//     Rule:
//             Foo Bar // Case 0
//     |       Foo Baz // Case 1
//
// The generated type will be something like
//
//     type Rule struct {
//             Case in // In [0, 1].
//             Bar  *Bar
//             Baz  *Baz
//             Foo  *Foo
//     }
//
// In the above, Foo and Bar fields will be non nill when Case is 0 and Foo and Baz fields will be non nil when Case is 1.
//
// The above holds when both Foo and Bar are non terminal symbols. If the production(s) contain also terminal symbols, all those symbols are turned into fields named Token with an optional numeric suffix when more than one non terminal appears in any of the production(s).
//
//     Rule:
//             Foo '+' Bar
//     |       Foo '[' NUMBER ']' Bar
//
// The generated type will be like
//
//     type Rule struct {
//             Case   int	// In [0, 1].
//             Bar    *Bar
//             Baz    *Baz
//             Foo    *Foo
//             Token  MyTokenType
//             Token2 MyTokenType
//             Token3 MyTokenType
//     }
//
// In the above, Token will capture '+' when Case is 0. For Case 1, Token will capture '[', Token2 NUMBER and Token3 ']'.
//
// MyTokenType is the type defined in the yacc %union as in
//
//     %union {
//             node    MyNodeType
//             Token   MyTokenType
//     }
//
// It is assumed that the lexer passed as an argument to yyParse instantiantes the lval.Token field with additional token information, like the lexeme value, starting position in the file etc.
//
// Generated actions
//
// There's a direct mapping, though not in the same order, of yacc pseudo variables $1, $2, ... and fields of the generated node types. For every production not disabled by the yy:ignore direction, yy injects code for instantiating the AST node when the production is reduced. For example, this rule from input.y
//
//     File:
//             Prologue TopLevelDeclList
//
// having no semantic action is turned into
//
//     File:
//             Prologue TopLevelDeclList
//             {
//                     $$ = &File{
//                             Prologue:          $1.(*Prologue),
//                             TopLevelDeclList:  $2.(*TopLevelDeclList).reverse(),
//                     }
//             }
//
// in output.y. The default yacc type of AST nodes is 'node' and can be changed  using the -node flag.
//
// Conventions
//
// Option-like rules, for example as in
//
//     BlockOpt:
//     |       Block
//
//     are converted into
//
//             BlockOpt:
//                     /* empty */
//                     {
//                             $$ = (*BlockOpt)(nil)
//                     }
//             |       Block
//                     {
//                             $$ = &BlockOpt{
//                                     Block:  $1.(*Block),
//                             }
//                     }
//
// in output.y, ie. the empty case does not produce a &RuleOpt{}, but nil instead to  conserve space.
//
// Generated examples depend on an user supplied function, by default named  exampleAST, with a signature
//
//     exampleAST(rule int, src string) interface{}
//
// This function is called with the production number, as assigned by goyacc and an example string generated by yy. exampleAST should parse the example string and return the AST created when production rule is reduced.
//
// When the project's parser is not yet working, a dummy exampleAST function  returnin always nil is a workaround.
//
// Magic names
//
// yy inspects rule actions found in the input file. If the action code  mentions identifier lx, yy asumes it refers to the yyLexer passed to  yyParse. In that case code like
//
//     lx := yylex.(*lexer)
//
// is injected near the beginning of the semantic action. The specific type  into which the yylex parameter is type asserted is adjustable using the  -yylex flag.  Similarly, when identifier lhs is mentioned, a short variable  definiton of variable lhs, like
//
//     lhs := &Foo{...}
//     $$ = lhs
//
// is injected into the output.y action, replacing the default generated action  (see "Concepts")
//
// For example, an action in input.y
//
//     |       IdentifierList Type '=' ExpressionList
//             {
//                     lhs.declare(lx.scope)
//             }
//
// Produces
//
//     {
//             lx := yylex.(*lexer)
//             lhs := &VarSpec{
//                     Case:            2,
//                     IdentifierList:  $1.(*IdentifierList).reverse(),
//                     Type:            $2.(*Type),
//                     Token:           $3,
//                     ExpressionList:  $4.(*ExpressionList).reverse(),
//             }
//             $$ = lhs
//             lhs.declare(lx.scope)
//     }
//
// in output.y.
//
// The AST examples generator depends on presence of the yy:token directive for  all non constant terminal symbols or the presence of the constant token  value as in this example
//
//     %token /*yy:token "%c" */       IDENTIFIER      "identifier"
//     %token                          BREAK           "break"
//
// Using fe
//
// The AST examples yy generates must be post processed by using the fe command  (http://godoc.org/github.com/cznic/fe), for example
//
//     $ go test -run ^Example[^_] | fe
//
// One of the reasons why this is not done automatically by yy is that the  above command will succeed only after your project has a _working_ scanner/parser combination. That's not the case in the early stages.
//
// Directives
//
// yy recognizes specially formatted comments within the input as directives. All  directive have the format
//
//     //yy:command argument
//
//  or
//
//     /*yy:command argument */
//
// Note that the directive must follow immediately the comment opening. There  must be no empty line(s) between the directive and the production it aplies  to.
//
// Directive example
//
// For example
//
//     //yy:example "foo * bar"
//     Rule:
//             Foo '*' Bar
//     //yy:example "foo / bar"
//     |       Foo '/' Bar
//
// The argument of the example directive is a doubly quoted Go string. The  string is used instead of an automatically generated example.
//
// Directive field
//
// For example
//
//     //yy:field      count   int
//     //yy:field      flag    bool
//     Rule: Foo Bar
//
// The argument of the field directive is the text up to the end of the  comment. The argument is added to the automatically generated fields of the  node type of Rule.
//
// Directive ignore
//
// For example
//
//     //yy:ignore
//     Rule: Foo Bar
//
// The ignore directive has no arguments. The directive disables generating of  the node type of Rule as well as generating code instantiating such node.
//
// Directive list
//
// For example
//
//     //yy:list
//     Rule:
//             Item
//     |       Rule ',' Item
//
// The list directive has no arguments. yy by default detects all left  recursive rules. When such rule has name having suffix 'List', yy  automatically generates proper reversing of the rule items. Using the list  directive enables the same when such a left recursive rule does not have  suffix 'List' in its name.
//
// Directive token
//
// For example
//
//     /*yy:token %c*/ IDENT
//     /*yy:token %d*/ NUMBER
//
// The argument of the token directive is a doubly quoted Go string. The string  is passed to a fmt.Sprinf call with an numeric argument chosen by yy that  falls small ASCII letters. The resulting string is used to generate textual  token values in examples.
//
// Directive case
//
// For example
//
//     //yy:case Foo
//     /*yy:case Bar */ NUMBER
//
// The argument of the case directive is an identifier, which is appended to
// the rule name to produce a symbolic and typed case number value. The type
// name is <RuleName>Case.
//
// Links
//
// Referenced from elsewhere:
//
//     [0]: https://golang.org/pkg/testing/#hdr-Examples
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/scanner"
	"go/token"
	"io"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/cznic/mathutil"
	yparser "github.com/cznic/parser/yacc"
	"github.com/cznic/strutil"
	"github.com/cznic/y"
)

const (
	caution  = "// Code generated by yy. DO NOT EDIT.\n\n"
	magicLx  = "lx"
	magicLHS = "lhs"
)

var (
	oAST          = flag.String("ast", "ast.go", "Output AST nodes definitions.")
	oASTExamples  = flag.String("astExamples", "ast_test.go", "Output AST examples.")
	oASTImport    = flag.String("astImport", "", "Optional AST file imports.")
	oExAST        = flag.String("exampleAST", "exampleAST", "Fuction to call to produce example ASTs.")
	oKind         = flag.String("kind", "kind", "Default node kind (rule case) field name.")
	oNode         = flag.String("node", "node", "Default non terminal yacc type.")
	oO            = flag.String("o", "parser.y", "Output yacc file.")
	oPkg          = flag.String("pkg", "", "Package name of generated Go files. Extract from input when blank.")
	oPrettyString = flag.String("prettyString", "prettyString", "Fuction to stringify things nicely.")
	oReport       = flag.String("v", "y.output", "create grammar report")
	oToken        = flag.String("token", "Token", "Default terminal yacc type.")
	oTokenSep     = flag.String("tokenSep", " ", "AST examples token separator string.")
	oYylex        = flag.String("yylex", "*lexer", "Type of yacc's yylex.")

	copyright       string                      // Extracted from input.
	fset            *token.FileSet              //
	kind            = map[*y.Rule]int{}         //
	nodes           = map[string]*node{}        // non terminal name: *node
	nonTerminals    []*y.Symbol                 // Sorted by name
	ruleDirectives  = map[*y.Rule][]directive{} //
	terminals       []*y.Symbol                 // Sorted by name
	tokenDirectives = map[string][]directive{}  // symbol name: []directive
	typed           = map[*y.Symbol]bool{}
	ytypes          = map[string]string{} // yacc type: Go type
)

type directive struct {
	cmd string
	arg string
}

func (d directive) field() (nm []string, typ string) {
	if d.cmd != "field" {
		panic("internal error")
	}

	s := strings.TrimSpace(d.arg)
	i := strings.IndexAny(s, " \t")
	if i < 0 {
		return nil, s
	}

	//TODO properly parse
	flds := s[:i]
	typ = s[i:]
	a := strings.Split(flds, ",")
	for _, v := range a {
		nm = append(nm, strings.TrimSpace(v))
	}
	return nm, typ
}

func (d *directive) isIgnore() bool { return d.cmd == "ignore" }
func (d *directive) isList() bool   { return d.cmd == "list" }

func unquote(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return ""
	}

	if c := s[0]; c != '"' && c != '\'' {
		return s
	}

	s2, err := strconv.Unquote(s)
	if err != nil {
		log.Fatalf("unquote %s: %v", s, err)
	}

	return s2
}

type node struct {
	fields map[string]string // name: Go type
	ytyp   string
	typ    string // Go type
}

func newNode(ytyp, typ string) *node { return &node{fields: map[string]string{}, ytyp: ytyp, typ: typ} }

func main() {
	log.SetFlags(0)
	flag.Parse()
	checkOptions()

	var rep io.Writer
	if nm := *oReport; nm != "" {
		f, err := os.Create(nm)
		if err != nil {
			log.Fatal(err)
		}

		defer f.Close()

		w := bufio.NewWriter(f)

		defer w.Flush()

		rep = w
	}

	fset = token.NewFileSet()
	spec, err := y.ProcessFile(fset, flag.Arg(0), &y.Options{
		AllowTypeErrors: true,
		Closures:        true,
		Report:          rep,
	})
	if err != nil {
		log.Fatal(err)
	}

	if *oPkg == "" {
		s := extractPkg(spec.Prologue)
		if s == "" {
			s = "main"
		}
		*oPkg = s
	}
	copyright = extractCopyright(spec.Prologue)
	union := spec.Union
	if union == nil {
		log.Fatalf("missing %sunion definition", "%")
	}

	for _, fields := range union.Fields.List[1:] { // Skip yys int.
		for _, nm := range fields.Names {
			ytypes[nm.Name] = astString(fields.Type)
		}
	}

	if _, ok := ytypes[*oNode]; !ok {
		log.Fatalf("undefined yacc type: %s", *oNode)
	}

	if _, ok := ytypes[*oToken]; !ok {
		log.Fatalf("undefined yacc type: %s", *oToken)
	}

	var a []string
	for _, sym := range spec.Syms {
		switch {
		case sym.IsTerminal:
			s := sym.Name
			s0 := s[0]
			if s == "error" || s0 > 0x7f || s0 == '#' || s0 == '$' {
				continue
			}

			a = append(a, sym.Name)
		default:
			if sym.Name[0] == '$' {
				break
			}

			a = append(a, sym.Name)
			for i, v := range sym.Rules {
				kind[v] = i
			}
		}
	}
	sort.Strings(a)
	for _, nm := range a {
		sym := spec.Syms[nm]
		if sym.IsTerminal {
			terminals = append(terminals, sym)
			continue
		}
		nonTerminals = append(nonTerminals, sym)
	}

	checkDirectives(spec)
	for _, v := range spec.Rules {
	out:
		for _, d := range ruleDirectives[v] {
			if d.cmd == "case" {
				typed[v.Sym] = true
				break out
			}
		}
	}
	nspec := genY(spec)
	genAST(spec)
	if *oASTExamples == os.DevNull {
		return
	}

	genASTExamples(nspec)
}

func checkOptions() {
	*oAST = strings.TrimSpace(*oAST)
	if *oAST == "" {
		*oAST = os.DevNull
	}

	*oASTExamples = strings.TrimSpace(*oASTExamples)
	if *oASTExamples == "" {
		*oASTExamples = os.DevNull
	}

	*oASTImport = strings.TrimSpace(*oASTImport)

	*oExAST = strings.TrimSpace(*oExAST)
	if *oExAST == "" {
		log.Fatal("invalid -exampleAST option value.")
	}

	*oKind = strings.TrimSpace(*oKind)
	if *oKind == "" {
		log.Fatal("invalid -kind option value.")
	}

	*oYylex = strings.TrimSpace(*oYylex)
	if *oYylex == "" {
		log.Fatal("invalid -yylex option value.")
	}

	*oNode = strings.TrimSpace(*oNode)
	if *oNode == "" {
		log.Fatal("invalid -node option value.")
	}

	*oO = strings.TrimSpace(*oO)
	if *oO == "" {
		*oO = os.DevNull
	}

	*oPkg = strings.TrimSpace(*oPkg)

	*oPrettyString = strings.TrimSpace(*oPrettyString)
	if *oPrettyString == "" {
		log.Fatal("invalid -prettyString option value.")
	}

	*oToken = strings.TrimSpace(*oToken)
	if *oToken == "" {
		log.Fatal("invalid -token option value.")
	}

	if g, e := flag.NArg(), 1; g != e {
		log.Fatalf("expected number of arguments: %v", e)
	}

	in := flag.Arg(0)
	if *oO == in {
		log.Fatal("input and output yacc file names are the same.")
	}
}

func extractPkg(s string) string {
	f, err := parser.ParseFile(token.NewFileSet(), flag.Arg(0), s, parser.PackageClauseOnly)
	if err != nil {
		return ""
	}

	return f.Name.Name
}

func extractCopyright(s string) string {
	a := strings.Split(s, "\n")
	for i, v := range a {
		a[i] = strings.TrimSpace(v)
	}

	for i, v := range a {
		if strings.HasPrefix(v, "// Copyright") {
			j := i + 1
			for ; j < len(a); j++ {
				if !strings.HasPrefix(a[j], "//") {
					j++
					break
				}
			}
			return strings.Join(a[i:j], "\n")
		}
	}
	return ""
}

func astString(node ast.Node) string {
	var buf bytes.Buffer
	var f func(node ast.Node)
	f = func(node ast.Node) {
		switch x := node.(type) {
		case *ast.Ident:
			buf.WriteString(x.Name)
		case *ast.InterfaceType:
			buf.WriteString("interface{")
			for _, v := range x.Methods.List {
				f(v)
				buf.WriteByte(';')
			}
			buf.WriteByte('}')
		case *ast.SelectorExpr:
			f(x.X)
			buf.WriteByte('.')
			f(x.Sel)
		case *ast.StarExpr:
			buf.WriteByte('*')
			f(x.X)
		default:
			log.Fatalf("unsupported go/ast.Node type %T", x)
		}
	}

	f(node)
	return buf.String()
}

func extractDirectives(tok *yparser.Token) (r []directive) {
	for _, s := range tok.Comments {
		if strings.HasPrefix(s, "/*") {
			if !strings.HasSuffix(s, "*/") {
				panic("internal error")
			}

			s = s[len("/*") : len(s)-len("*/")]
			a := strings.Split(s, "\n")
			s = "//" + strings.Join(a, "\n//")
			a = strings.Split(s, "\n")
			r = append(r, extractDirectives(&yparser.Token{Comments: a, File: tok.File, Char: tok.Char})...)
			continue
		}

		if !strings.HasPrefix(s, "//yy:") {
			continue
		}

		s = s[len("//yy:"):]
		i := strings.IndexAny(s, " \t")
		if i < 0 {
			i = len(s)
		}
		cmd := s[:i]
		arg := strings.TrimSpace(s[i:])
		if _, ok := isCmd[cmd]; !ok {
			log.Fatalf("%v: invalid directive: %s", tok.Position(), cmd)
		}

		r = append(r, directive{cmd: cmd, arg: arg})
	}
	return r
}

var isCmd = map[string]struct{}{
	"case":    {},
	"example": {},
	"field":   {},
	"ignore":  {},
	"list":    {},
	"token":   {},
}

func checkDirectives(spec *y.Parser) {
	for _, rule := range spec.Rules[1:] {
		if rule.Sym.Name[0] == '$' {
			continue
		}

		d := extractDirectives(rule.Token)
		if len(d) == 0 {
			continue
		}

		ruleDirectives[rule] = d
	}
	for _, def := range spec.Definitions {
		if def.Case != 3 || def.ReservedWord.Token.Char.Rune != yparser.TOKEN {
			continue
		}

		for _, nm := range def.Nlist {
			d := extractDirectives(nm.Token)
			if len(d) == 0 {
				continue
			}

			tokenDirectives[nm.Token.Val] = d
		}
	}
}

func isIgnored(sym *y.Symbol) bool {
	for _, rule := range sym.Rules {
		for _, d := range ruleDirectives[rule] {
			if d.isIgnore() {
				return true
			}
		}
	}
	return false
}

func isList(sym *y.Symbol) bool {
	if !sym.IsLeftRecursive {
		return false
	}

	if strings.HasSuffix(sym.Name, "List") {
		return true
	}

	for _, rule := range sym.Rules {
		for _, d := range ruleDirectives[rule] {
			if d.isList() {
				return true
			}
		}
	}
	return false
}

func genAST(spec *y.Parser) {
	f := bytes.NewBuffer(nil)
	fmt.Fprintf(f, caution)
	if copyright != "" {
		fmt.Fprintf(f, "%s\n", copyright)
	}
	fmt.Fprintf(f, "package %s\n", *oPkg)
	if s := *oASTImport; s != "" {
		fmt.Fprintf(f, "\nimport(%s)\n", s)
	}
	var a []string
	for nm := range nodes {
		a = append(a, nm)
	}
	sort.Strings(a)
	for _, nm := range a {
		sym := spec.Syms[nm]
		if isIgnored(sym) {
			continue
		}

		if typed[sym] {
			fmt.Fprintf(f, "\n// %sCase represents case numbers of production %s\n", nm, nm)
			fmt.Fprintf(f, "type %sCase int\n", nm)
			fmt.Fprintf(f, "\n// Values of type %sCase\n", nm)
			fmt.Fprintf(f, "const (\n")
		outer:
			for i, rule := range sym.Rules {
				for _, d := range ruleDirectives[rule] {
					if d.cmd == "case" {
						switch {
						case i == 0:
							fmt.Fprintf(f, "%s%s %sCase = iota\n", nm, d.arg, nm)
						default:
							fmt.Fprintf(f, "%s%s\n", nm, d.arg)
						}
						continue outer
					}
				}
				fmt.Fprintln(f, "_")
			}
			fmt.Fprintf(f, ")\n")
			fmt.Fprintf(f, "\n// String implements fmt.Stringer\n")
			fmt.Fprintf(f, "func (n %sCase) String() string {\n", nm)
			fmt.Fprintf(f, "switch n {\n")
		outer2:
			for _, rule := range sym.Rules {
				for _, d := range ruleDirectives[rule] {
					if d.cmd == "case" {
						fmt.Fprintf(f, "case %s%s:\nreturn \"%s%s\"\n", nm, d.arg, nm, d.arg)
						continue outer2
					}
				}
			}
			fmt.Fprintf(f, "default:\nreturn fmt.Sprintf(\"%sCase(%%v)\", int(n))\n", nm)
			fmt.Fprintf(f, "}\n")
			fmt.Fprintf(f, "}\n")
		}

		plural := ""
		if len(sym.Rules) > 1 {
			plural = "s"
		}
		fmt.Fprintf(f, "\n// %s represents data reduced by production%s:\n//\n%s\n", nm, plural, symDocs(sym))
		n := nodes[nm]
		fmt.Fprintf(f, "type %s struct{\n", nm)
		var a []string
		for _, rule := range sym.Rules {
			d := ruleDirectives[rule]
			for _, d := range d {
				if d.cmd == "field" {
					fmt.Fprintf(f, "%s\n", d.arg)
				}
			}
		}
		for nm := range n.fields {
			a = append(a, nm)
		}
		sort.Strings(a)
		for _, nm := range a {
			fmt.Fprintf(f, "\t%s %s\n", nm, n.fields[nm])
		}
		fmt.Fprintf(f, "}\n")

		rx := "n"

		switch isList(sym) {
		case true:
			fmt.Fprintf(f, `
func (%[1]s *%[2]s) reverse() *%[2]s {
	if %[1]s == nil {
		return nil
	}

	na := %[1]s
	nb := na.%[2]s
	for nb != nil {
		nc := nb.%[2]s
		nb.%[2]s = na
		na = nb
		nb = nc
	}
	%[1]s.%[2]s = nil
	return na
}

func (%[1]s *%[2]s) fragment() interface{} { return %[1]s.reverse() }
`, rx, nm)
		default:
			fmt.Fprintf(f, `
func (%[1]s *%[2]s) fragment() interface{} { return %[1]s }
`, rx, nm)
		}

		fmt.Fprintf(f, `// String implements fmt.Stringer.
func (%[1]s *%[2]s) String() string {
	return %[3]s(%[1]s)
}
`, rx, nm, *oPrettyString)

		fmt.Fprintf(f, `// Pos reports the position of the first component of %[1]s or zero if it's empty.
func (%[1]s *%[2]s) Pos() token.Pos {
`, rx, nm)
		cases := map[string][]int{}
		isopt := isOpt(sym)
		fmt.Fprintf(f, "if %s == nil { return 0 }\n\n", rx)
		for i, v := range sym.Rules {
			m := map[string]int{}
			s := ""
			for _, c := range v.Components {
				if strings.HasPrefix(c, "$") {
					continue
				}

				isT := false
				cs := spec.Syms[c]
				if cs.IsTerminal {
					c = *oToken
					isT = true
				}
				m[c]++
				suffix := ""
				if m[c] > 1 {
					suffix = strconv.Itoa(m[c])
				}
				if s != "" {
					s += " "
				}
				s += fmt.Sprintf("%s$%s", c, suffix)
				if isT || !cs.DerivesEmpty() {
					break
				}
			}
			cases[s] = append(cases[s], i)
		}
		a = []string{}
		for k := range cases {
			a = append(a, k)
		}
		sort.Strings(a)
		if len(a) > 1 && !isopt {
			fmt.Fprintf(f, "switch %s.%s {\n", rx, *oKind)
		}
		for _, k := range a {
			v := cases[k]
			if len(a) > 1 && !isopt {
				fmt.Fprintf(f, "case ")
				for _, v := range v[:len(v)-1] {
					fmt.Fprintf(f, "%d, ", v)
				}
				fmt.Fprintf(f, "%d", v[len(v)-1])
				fmt.Fprintf(f, ":\n")
			}
			if k == "" {
				if !isopt {
					fmt.Fprintf(f, "return 0\n")
				}
				continue
			}

			a := strings.Split(k, " ")
			for _, v := range a[:len(a)-1] {
				v = strings.Replace(v, "$", "", -1)
				fmt.Fprintf(f, "if p := %s.%s.Pos(); p != 0 { return p }\n\n", rx, v)
			}
			s := strings.Replace(a[len(a)-1], "$", "", -1)
			fmt.Fprintf(f, "return %s.%s.Pos()\n", rx, s)
		}
		if len(a) > 1 && !isopt {
			fmt.Fprintf(f, "default:\n")
			fmt.Fprintf(f, "panic(\"internal error\")\n")
			fmt.Fprintf(f, "}\n")
		}
		fmt.Fprintf(f, "}\n")
		continue
	}

	b, err := format.Source(f.Bytes())
	if err != nil {
		b = f.Bytes()
	}

	file, err := os.Create(*oAST)
	if err != nil {
		log.Fatal(err)
	}

	if _, err := file.Write(b); err != nil {
		log.Fatal(err)
	}

	if err := file.Close(); err != nil {
		log.Fatal(err)
	}
}

func genNodes(spec *y.Parser) {
	for _, sym := range terminals {
		if sym.Type == "" {
			sym.Type = *oToken
		}
	}
	for _, sym := range nonTerminals {
		if sym.Type == "" {
			sym.Type = *oNode
		}
	}

	for _, sym := range nonTerminals {
		nm := sym.Name
		n := nodes[nm]
		if n == nil {
			ytyp := sym.Type
			if ytyp == "" {
				ytyp = *oNode
			}
			typ := fmt.Sprintf("*%s", nm)
			if ytyp != *oNode {
				typ = ytypes[ytyp]
			}
			n = newNode(ytyp, typ)
			if len(sym.Rules) > 1 && !isOpt(sym) {
				typ := "int"
				if typed[sym] {
					typ = fmt.Sprintf("%sCase", nm)
				}
				n.fields[*oKind] = typ
			}
			nodes[nm] = n
		}
	}
	for _, sym := range nonTerminals {
		nm := sym.Name
		n := nodes[nm]
		for _, rule := range sym.Rules {
			m := nodeFieldMap0(sym)
			for _, c := range rule.Components {
				if c[0] == '$' {
					continue
				}

				csym := spec.Syms[c]
				fldname := c
				if csym.IsTerminal {
					fldname = *oToken
				}
				typ := ytypes[fldname]
				if !csym.IsTerminal {
					typ = nodes[c].typ
				}
				m[fldname]++
				i := m[fldname]
				if i != 1 {
					fldname = fmt.Sprintf("%s%v", fldname, i)
				}
				n.fields[fldname] = typ
			}
		}
	}
}

func nodeFieldMap0(sym *y.Symbol) map[string]int {
	m := map[string]int{}
	if len(sym.Rules) != 0 {
		m[*oKind] = 1
	}
	for _, rule := range sym.Rules {
		d := ruleDirectives[rule]
		if len(d) == 0 {
			continue
		}
		for _, d := range d {
			if d.cmd == "field" {
				flds, typ := d.field()
				if len(flds) == 0 {
					m[typ] = 1
					continue
				}

				for _, v := range flds {
					m[v] = 1
				}
			}
		}
	}
	return m
}

func declareComponents(f io.Writer, spec *y.Parser, list []*y.Symbol, kind string) {
	m := map[string][]string{} // yacc type: []token name
	w := 0
	for _, v := range list {
		ytyp := v.Type
		m[ytyp] = append(m[ytyp], v.Name)
		s := ""
		if val := v.ExplicitValue; v.IsTerminal && val >= 0 && v.Name[0] != '\'' {
			s = fmt.Sprintf(" %d", val)
		}
		w = mathutil.Max(w, len(v.Name)+len(s))
	}
	var a []string
	for tn := range m {
		a = append(a, tn)
	}
	sort.Strings(a)
	for _, tn := range a {
		nms := m[tn]
		var a []string
		for _, nm := range nms {
			a = append(a, nm)
		}
		sort.Strings(a)
		fmt.Fprintf(f, "\n%s\t<%s>\n", kind, tn)
		for _, nm := range a {
			ls := ""
			sym := spec.Syms[nm]
			ns := ""
			if v := sym.ExplicitValue; sym.IsTerminal && v >= 0 && nm[0] != '\'' {
				ns = fmt.Sprintf(" %d", v)
			}
			if s := sym.LiteralString; s != "" {
				ls = strings.Repeat(" ", w-len(ns)+2-len(nm)) + s
			}
			fmt.Fprintf(f, "\t%s%s%s\n", nm, ns, ls)
		}
	}
}

func declareAssocDefs(f io.Writer, defs []*y.AssocDef) {
	if len(defs) != 0 {
		fmt.Fprintln(f)
	}
	for _, def := range defs {
		var s string
		switch def.Associativity {
		case y.AssocLeft:
			s = "%left"
		case y.AssocRight:
			s = "%right"
		case y.AssocNone:
			s = "%nonassoc"
		case y.AssocPrecedence:
			s = "%precedence"
		}
		fmt.Fprintf(f, "%s", s)
		sep := "\t"
		for _, sym := range def.Syms {
			fmt.Fprintf(f, "%s%s", sep, sym)
			sep = " "
		}
		fmt.Fprintln(f)
	}
}

func mentions(src string, ids map[string]bool) map[string]bool {
	var s scanner.Scanner
	s.Init(token.NewFileSet().AddFile("", -1, len(src)), []byte(src), nil, 0)
	for {
		_, tok, lit := s.Scan()
		if tok == token.EOF {
			return ids
		}

		if tok != token.IDENT {
			continue
		}

		if _, ok := ids[lit]; ok {
			ids[lit] = true
		}
	}
}

func litSym(spec *y.Parser, s string) string {
	if s[0] != '"' {
		return s
	}

	x := spec.LiteralStrings[s]
	if x == nil {
		return s
	}

	return x.Name
}

func genY(spec *y.Parser) (nspec *y.Parser) {
	f := bytes.NewBuffer(nil)

	defer func() {
		file, err := os.Create(*oO)
		if err != nil {
			log.Fatal(err)
		}

		if _, err := file.Write(f.Bytes()); err != nil {
			log.Fatal(err)
		}

		if err := file.Close(); err != nil {
			log.Fatal(err)
		}

		buf := f.Bytes()
		fset.AddFile(*oO, -1, len(buf))
		if nspec, err = y.ProcessSource(fset, *oO, buf, &y.Options{}); err != nil {
			log.Fatal(err)
		}
	}()

	fmt.Fprintf(f, caution)
	if s := spec.Prologue; s != "" {
		fmt.Fprintf(f, "%%{%s%%}", s)
	}

	a := []string{"type t struct {"}
	var buf bytes.Buffer
	for _, fields := range spec.Union.Fields.List[1:] { // Skip yys int.
		buf.Reset()
		for i, v := range fields.Names {
			if i != 0 {
				buf.WriteByte(',')
			}
			buf.WriteString(astString(v))
		}
		buf.WriteByte(' ')
		buf.WriteString(astString(fields.Type))
		buf.WriteByte(';')
		a = append(a, buf.String())
	}
	a = append(a, "}")
	s := strings.Join(a, "\n")
	b, err := format.Source([]byte(s))
	if err == nil {
		s = string(b)
	}
	fmt.Fprintf(f, "\n\n%%union%s\n", s[len("type t struct"):])
	genNodes(spec)
	//TODO inline NT: T;

	declareComponents(f, spec, terminals, "%token")
	declareComponents(f, spec, nonTerminals, "%type")
	declareAssocDefs(f, spec.AssocDefs)
	if spec.ErrorVerbose {
		fmt.Fprintf(f, "\n%%error-verbose\n")
	}
	fmt.Fprintf(f, "\n%%start %s\n\n%%%%\n", spec.Start)

	rnm := ""
	var clist []string
	for _, rule := range spec.Rules[1:] { // Skip rule 0
		nm := rule.Sym.Name
		if nm[0] == '$' {
			continue
		}

		clist = clist[:0]
		prec := ""
		if sym := rule.ExplicitPrecSym; sym != nil {
			prec = fmt.Sprintf(" %%prec %s", sym)
		}
		if nm != rnm {
			rnm = nm
			fmt.Fprintf(f, "\n%s:\n", nm)
		} else {
			fmt.Fprintf(f, "|")
		}
		body := normalizedBody(rule.Body)
		sep := "\t"
		act := false
		for i, v := range body {
			switch x := v.(type) {
			case int:
				nm := fmt.Sprintf("%q", x)
				clist = append(clist, nm)
				fmt.Fprintf(f, "%s%s", sep, nm)
				sep = " "
				act = false
			case string:
				x0 := x
				x = litSym(spec, x)
				clist = append(clist, x)
				fmt.Fprintf(f, "%s%s", sep, x0)
				sep = " "
				act = false
			case *yparser.Action:
				isSemanticAction := i == len(body)-1
				if isSemanticAction {
					if len(clist) == 0 {
						fmt.Fprintf(f, "\t/* empty */%s\n", prec)
					} else {
						fmt.Fprintf(f, "%s", prec)
					}
				}
				clist = append(clist, "")
				f2 := bytes.NewBuffer(nil)
				for _, v := range x.Values {
					f2.WriteString(v.Src)
				}
				s := f2.String()
				s = constructNode(
					kind[rule],
					spec,
					isSemanticAction,
					s,
					clist,
					mentions(s, map[string]bool{magicLx: false, magicLHS: false}),
					rule,
				)
				if sep != "\t" {
					fmt.Fprintln(f)
				}
				fmt.Fprintf(f, "\t%s\n", s)
				sep = "\t"
				act = true
			}
		}
		if !act {
			fmt.Fprintln(f)
		}
	}

	if s := spec.Tail; s != "" {
		fmt.Fprintf(f, "\n%%%%%s", s)
	}
	return nil
}

func normalizedBody(body []interface{}) []interface{} {
	body = append([]interface{}(nil), body...)
	switch n := len(body); n {
	default:
		if _, ok := body[n-1].(*yparser.Action); ok {
			return body
		}

		fallthrough
	case 0:
		return append(body, &yparser.Action{
			Values: []*yparser.ActionValue{{Src: "{\n\t}", Type: yparser.ActionValueGo}},
		})
	}
}

func constructNode(kind int, spec *y.Parser, semanticAction bool, src string, clist []string, ids map[string]bool, rule *y.Rule) string {
	sym := rule.Sym
	caseName := ""
	for _, d := range ruleDirectives[rule] {
		if d.cmd == "case" {
			caseName = sym.Name + d.arg
		}
	}
	ignored := isIgnored(sym)
	n := nodes[sym.Name]
	inj := ""
	final := ""
	i := strings.Index(src, "{")
	if src[i+1] != '\n' {
		final = "\n\t\t"
	}
	if ids[magicLx] {
		inj = fmt.Sprintf("\n\t\tlx := yylex.(%s)", *oYylex)
	}
	switch {
	case !semanticAction:
		// ok
	case len(clist) == 0 || len(clist) == 1 && clist[0] == "":
		if ignored {
			break
		}

		inj += fmt.Sprintf("\n\t\t$$ = (%s)(nil)", n.typ)
	default:
		var a []string
		if !ignored {
			if ids[magicLHS] {
				a = append(a, fmt.Sprintf("\n\t\tlhs := &%s{", n.typ[1:]))
			} else {
				a = append(a, fmt.Sprintf("\n\t\t$$ = &%s{", n.typ[1:]))
			}
		}

		m := nodeFieldMap0(sym)
		if !ignored {
			if (kind != 0 || caseName != "") && !isOpt(sym) {
				k := interface{}(kind)
				if caseName != "" {
					k = caseName
				}
				a = append(a, fmt.Sprintf("\t%v: %v,", *oKind, k))
			}
		}
		if !ignored {
			for i, c := range clist {
				if c == "" || c == "error" {
					continue
				}

				csym := spec.Syms[c]
				fldname := c
				if csym.IsTerminal {
					fldname = *oToken
				}
				typ := ytypes[fldname]
				//TODO- if !csym.IsTerminal {
				//TODO- 	typ = fmt.Sprintf("*%s", c)
				//TODO- }
				if typ == "" {
					typ = fmt.Sprintf("*%s", c)
				}
				assert := fmt.Sprintf(".(%s)", typ)
				if !csym.IsTerminal {
					typ = nodes[c].typ
					if csym.Type != *oNode {
						assert = ""
					}
				}
				m[fldname]++
				j := m[fldname]
				if j != 1 {
					fldname = fmt.Sprintf("%s%v", fldname, j)
				}
				ta := ""
				if !csym.IsTerminal {
					re := ""
					if isList(csym) && csym != sym {
						re = ".reverse()"
					}
					ta = fmt.Sprintf("%s%s", assert, re)
				}
				a = append(a, fmt.Sprintf("\t%s: $%d%s,", fldname, i+1, ta))
			}
			w := 0
			for _, v := range a[1:] {
				i := strings.Index(v, ":")
				w = mathutil.Max(w, i)
			}
			for i, v := range a[1:] {
				x := strings.SplitAfterN(v, ":", 2)
				a[i+1] = x[0] + strings.Repeat(" ", w+2-len(x[0])) + x[1]
			}
			a = append(a, "}")
		}
		if ids[magicLHS] {
			a = append(a, "$$ = lhs")
		}
		inj += strings.Join(a, "\n\t\t")
	}
	return src[:i+1] + fmt.Sprintf("%s%s", inj, final) + src[i+1:]
}

func symDocs(sym *y.Symbol) string {
	nm := sym.Name
	a := []string{"//\t" + nm + ":"}
	pref := "        "
	w := 0
	for _, rule := range sym.Rules {
		s := ""
		for _, v := range rule.Body {
			switch x := v.(type) {
			case int:
				s += fmt.Sprintf("%s%q", pref, x)
			case string:
				s += pref + x
			default:
				continue
			}

			pref = " "
		}
		if s == "" {
			s = pref + "/* empty */"
		}
		w = mathutil.Max(w, len(s))
		a = append(a, s)
		pref = "|       "
	}
	for i := 1; i < len(a); i++ {
		k := interface{}(i - 1)
		for _, d := range ruleDirectives[sym.Rules[i-1]] {
			if d.cmd == "case" {
				k = nm + d.arg
				break
			}
		}
		s := a[i]
		s += fmt.Sprintf("%s// %s %v", strings.Repeat(" ", w+2-len(s)), *oKind, k)
		a[i] = s
	}
	return strings.Join(a, "\n//\t")
}

const lits = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

var ilit int

func genASTExamples(spec *y.Parser) {
	f0 := bytes.NewBuffer(nil)
	f := strutil.IndentFormatter(f0, "\t")

	defer func() {
		file, err := os.Create(*oASTExamples)
		if err != nil {
			log.Fatal(err)
		}

		b, err := format.Source(f0.Bytes())
		if err != nil {
			b = f0.Bytes()
		}
		if _, err := file.Write(b); err != nil {
			log.Fatal(err)
		}

		if err := file.Close(); err != nil {
			log.Fatal(err)
		}
	}()

	fmt.Fprintf(f, caution)
	if copyright != "" {
		fmt.Fprintf(f, "%s\n", copyright)
	}
	fmt.Fprintf(f, `package %s

import (
	"fmt"
)
`, *oPkg)

	states := spec.Reductions()
	for _, sym := range nonTerminals {
		nm := sym.Name
		rules := sym.Rules
		w := len(strconv.Itoa(len(rules)))
	nextExample:
		for icase, rule := range rules {
			if isIgnored(rule.Sym) {
				continue
			}

			for _, c := range rule.Components {
				if c == "error" {
					continue nextExample
				}
			}

			src := ""
			for _, d := range ruleDirectives[rule] {
				if d.cmd != "example" {
					continue
				}

				src = unquote(d.arg)
				break
			}

			ruleNum := rule.RuleNum
			if src == "" {
				ilit = 0
				var r []string
				state, ok := states[ruleNum]
				if !ok { // State not reachable?
					continue nextExample
					//TODO- panic("internal error")
				}

				syms := spec.States[state[0]].Reduce0(rule)
				if len(syms) == 0 {
					log.Fatalf("rule %d not reduced in state %d", ruleNum, state[0])
				}

				for _, v := range syms {
					nm := v.Name
					if nm == "$end" {
						continue
					}

					if nm == "error" {
						continue nextExample
					}

					ls := v.LiteralString
					tokDirective := false
					for _, d := range tokenDirectives[nm] {
						if d.cmd != "token" {
							continue
						}

						ls = unquote(d.arg)
						tokDirective = true
						break
					}
					if ls == "" {
						if nm[0] == '\'' {
							ls = unquote(nm)
						} else {
							log.Fatalf("%v: no literal string for %s", fset.Position(v.Pos), nm)
						}
					}
					if tokDirective {
						s := strings.Replace(ls, "%%", "", -1)
						if strings.Contains(s, "%") {
							ls = fmt.Sprintf(ls, lits[ilit])
							ilit = (ilit + 1) % len(lits)
						}
					} else {
						ls = unquote(ls)
					}
					r = append(r, strings.Trim(ls, " \t\v\f"))
				}
				src = strings.Join(r, *oTokenSep)
			}

			ok := false
			for _, v := range ruleDirectives[rule] {
				if v.cmd == "case" {
					s := strings.ToLower(v.arg[:1]) + v.arg[1:]
					f.Format("\nfunc Example%s_%v() {%i\n", nm, s)
					ok = true
					break
				}
			}
			if !ok {
				f.Format("\nfunc Example%s_case%0*d() {%i\n", nm, w, icase)
			}
			switch len(normalizedBody(rule.Body)) {
			case 1:
				f.Format("fmt.Println(%s(%d, %q) == (%s)(nil))\n", *oExAST, ruleNum, src, nodes[nm].typ)
			default:
				f.Format("fmt.Println(%s(%d, %q))\n", *oExAST, ruleNum, src)
			}
			f.Format("// Output:\n%u}\n")
		}
	}
}

func isOpt(sym *y.Symbol) bool {
	if sym.IsTerminal || len(sym.Rules) != 2 {
		return false
	}

	r1, r2 := sym.Rules[0], sym.Rules[1]
	if len(r1.Components) != 0 {
		r1, r2 = r2, r1
	}

	return len(r1.Components) == 0 && len(r2.Components) != 0
}