Esempio n. 1
0
// Read returns the next Source ScannerRune.
func (s *Source) Read() (r ScannerRune) {
	for {
		r.Position = s.Position()
		r.Rune, r.Size, r.Err = s.tos.reader.ReadRune()
		if r.Err == nil || !fileutil.IsEOF(r.Err) {
			p := &s.tos.position
			p.Offset += r.Size
			if r.Rune != '\n' {
				p.Column++
			} else {
				p.Line++
				p.Column = 1
			}
			return
		}

		// err == os.EOF, try parent source
		if sp := len(s.stack) - 1; sp >= 0 {
			s.tos = s.stack[sp]
			s.stack = s.stack[:sp]
		} else {
			r.Rune, r.Size = 0, 0
			return
		}
	}
}
Esempio n. 2
0
// NewL parses a .l source fname from src, returns L or an error if any.
// Currently it is not reentrant and not invokable more than once in an application
// (which is assumed tolerable for a "lex" tool).
// The unoptdfa argument allows to disable optimization of the produced DFA.
// The mode32 parameter is not yet supported and must be false.
func NewL(fname string, src io.RuneReader, unoptdfa, mode32 bool) (l *L, err error) {
	if mode32 {
		return nil, errors.New("lex.NewL: mode32 unsupported yet")
	}

	nodfaopt, bits32 = unoptdfa, mode32
	l = &L{}

	if !hook {
		defer func() {
			if e := recover(); e != nil {
				l = nil
				err = e.(error)
			}
		}()
	}

	// Support \r\n line separators too
	in := []rune{}
loop:
	for {
		r, _, err := src.ReadRune()
		switch {
		case err == nil:
			in = append(in, r)
		case fileutil.IsEOF(err):
			break loop
		default:
			return nil, err
		}
	}
	src = bytes.NewBufferString(strings.Replace(string(in), "\r\n", "\n", -1))

	scanner := lxr.Scanner(fname, src)
	if y := yyParse(newTokenizer(scanner)); y != 0 || len(errors_) != 0 {
		return nil, errors.New(strings.Join(errors_, "\n"))
	}

	computePartialDFAs()
	if len(errors_) != 0 {
		return nil, errors.New(strings.Join(errors_, "\n"))
	}

	computeAllNfa()
	allDfa = allNfa.powerSet()
	for _, irule := range allDfa.acceptRule {
		delete(unreachableRules, irule)
	}
	for irule := range unreachableRules {
		logErr(fmt.Sprintf("%s - pattern `%s` unreachable", rulePos[irule], rules[irule].pattern))
	}
	if len(errors_) != 0 {
		return nil, errors.New(strings.Join(errors_, "\n"))
	}

	l.DefCode = defCode
	l.UserCode = usrCode
	l.StartConditions = iStarts
	l.StartConditionsStates = make(map[int]*lexer.NfaState)
	l.StartConditionsBolStates = make(map[int]*lexer.NfaState)
	for _, edge0 := range allDfa.nfa.in.Consuming {
		switch edge := edge0.(type) {
		default:
			panic(errors.New("internal error"))
		case *lexer.RuneEdge:
			if _, ok := l.StartConditionsStates[int(edge.Rune)]; ok {
				panic(errors.New("internal error"))
			}
			if edge.Rune < 128 {
				l.StartConditionsStates[int(edge.Rune)] = edge.Target()
			} else {
				l.StartConditionsBolStates[int(edge.Rune)-128] = edge.Target()
			}
		case *lexer.RangesEdge:
			for _, rng := range edge.Ranges.R32 {
				for arune := rng.Lo; arune <= rng.Hi; arune += rng.Stride {
					if _, ok := l.StartConditionsStates[int(arune)]; ok {
						panic(errors.New("internal error"))
					}
					if arune < 128 {
						l.StartConditionsStates[int(arune)] = edge.Target()
					} else {
						l.StartConditionsBolStates[int(arune)-128] = edge.Target()
					}
				}
			}
		}

	}
	for _, rule := range rules {
		l.Rules = append(l.Rules, Rule{Conds: rule.conds, Pattern: rule.pattern, RE: rule.re, Action: rule.action, BOL: rule.bol, EOL: rule.eol})
	}
	l.Dfa = allDfa.nfa.nfa[1:]
	l.Accepts = map[*lexer.NfaState]int{}
	for id, state := range allDfa.accept {
		l.Accepts[state] = allDfa.acceptRule[id]
	}
	l.YYT = _yyt
	l.YYB = _yyb
	l.YYC = _yyc
	l.YYN = _yyn
	l.YYM = _yym
	return
}