// Read returns the next Source ScannerRune. func (s *Source) Read() (r ScannerRune) { for { r.Position = s.Position() r.Rune, r.Size, r.Err = s.tos.reader.ReadRune() if r.Err == nil || !fileutil.IsEOF(r.Err) { p := &s.tos.position p.Offset += r.Size if r.Rune != '\n' { p.Column++ } else { p.Line++ p.Column = 1 } return } // err == os.EOF, try parent source if sp := len(s.stack) - 1; sp >= 0 { s.tos = s.stack[sp] s.stack = s.stack[:sp] } else { r.Rune, r.Size = 0, 0 return } } }
// NewL parses a .l source fname from src, returns L or an error if any. // Currently it is not reentrant and not invokable more than once in an application // (which is assumed tolerable for a "lex" tool). // The unoptdfa argument allows to disable optimization of the produced DFA. // The mode32 parameter is not yet supported and must be false. func NewL(fname string, src io.RuneReader, unoptdfa, mode32 bool) (l *L, err error) { if mode32 { return nil, errors.New("lex.NewL: mode32 unsupported yet") } nodfaopt, bits32 = unoptdfa, mode32 l = &L{} if !hook { defer func() { if e := recover(); e != nil { l = nil err = e.(error) } }() } // Support \r\n line separators too in := []rune{} loop: for { r, _, err := src.ReadRune() switch { case err == nil: in = append(in, r) case fileutil.IsEOF(err): break loop default: return nil, err } } src = bytes.NewBufferString(strings.Replace(string(in), "\r\n", "\n", -1)) scanner := lxr.Scanner(fname, src) if y := yyParse(newTokenizer(scanner)); y != 0 || len(errors_) != 0 { return nil, errors.New(strings.Join(errors_, "\n")) } computePartialDFAs() if len(errors_) != 0 { return nil, errors.New(strings.Join(errors_, "\n")) } computeAllNfa() allDfa = allNfa.powerSet() for _, irule := range allDfa.acceptRule { delete(unreachableRules, irule) } for irule := range unreachableRules { logErr(fmt.Sprintf("%s - pattern `%s` unreachable", rulePos[irule], rules[irule].pattern)) } if len(errors_) != 0 { return nil, errors.New(strings.Join(errors_, "\n")) } l.DefCode = defCode l.UserCode = usrCode l.StartConditions = iStarts l.StartConditionsStates = make(map[int]*lexer.NfaState) l.StartConditionsBolStates = make(map[int]*lexer.NfaState) for _, edge0 := range allDfa.nfa.in.Consuming { switch edge := edge0.(type) { default: panic(errors.New("internal error")) case *lexer.RuneEdge: if _, ok := l.StartConditionsStates[int(edge.Rune)]; ok { panic(errors.New("internal error")) } if edge.Rune < 128 { l.StartConditionsStates[int(edge.Rune)] = edge.Target() } else { l.StartConditionsBolStates[int(edge.Rune)-128] = edge.Target() } case *lexer.RangesEdge: for _, rng := range edge.Ranges.R32 { for arune := rng.Lo; arune <= rng.Hi; arune += rng.Stride { if _, ok := l.StartConditionsStates[int(arune)]; ok { panic(errors.New("internal error")) } if arune < 128 { l.StartConditionsStates[int(arune)] = edge.Target() } else { l.StartConditionsBolStates[int(arune)-128] = edge.Target() } } } } } for _, rule := range rules { l.Rules = append(l.Rules, Rule{Conds: rule.conds, Pattern: rule.pattern, RE: rule.re, Action: rule.action, BOL: rule.bol, EOL: rule.eol}) } l.Dfa = allDfa.nfa.nfa[1:] l.Accepts = map[*lexer.NfaState]int{} for id, state := range allDfa.accept { l.Accepts[state] = allDfa.acceptRule[id] } l.YYT = _yyt l.YYB = _yyb l.YYC = _yyc l.YYN = _yyn l.YYM = _yym return }