Beispiel #1
0
func (it ngramSection) Next(line []byte) (stream.Iteratee, bool, error) {
	if line[0] != '\\' || !bytes.HasSuffix(line, []byte("-grams:")) {
		return nil, false, stream.ErrExpect(`section header "\N-grams:"`)
	}
	n, err := strconv.Atoi(string(line[1 : len(line)-len("-grams:")]))
	if err != nil || n <= 0 {
		return nil, false, stream.ErrExpect(`positive integer in section header "\N-grams:"`)
	}
	return newNgramEntries(n, it.builder), true, nil
}
Beispiel #2
0
func (it *ngramEntries) setParts(line []byte) error {
	// p
	x, xs := tokenSplit(line)
	if x == "" {
		return stream.ErrExpect("log-probability")
	}
	if f, err := strconv.ParseFloat(x, WEIGHT_SIZE); err != nil {
		return err
	} else {
		it.p = Weight(f)
	}
	// context
	for i := 1; i < it.n; i++ {
		x, xs = tokenSplit(xs)
		if x == "" {
			return stream.ErrExpect(fmt.Sprintf("%d context word(s)", it.n))
		}
		it.context[i-1] = x
	}
	// word
	x, xs = tokenSplit(xs)
	if x == "" {
		return stream.ErrExpect("word")
	}
	it.word = x
	// bow
	x, xs = tokenSplit(xs)
	if x == "" {
		it.bow = 0
	} else if f, err := strconv.ParseFloat(x, WEIGHT_SIZE); err == nil {
		it.bow = Weight(f)
	} else {
		return err
	}
	// no extra stuff
	if len(xs) != 0 {
		return stream.ErrExpect("end of line")
	}
	return nil
}
Beispiel #3
0
func (it ngramSection) Final() error { return stream.ErrExpect(`\N-grams: ...`) }