func (it ngramSection) Next(line []byte) (stream.Iteratee, bool, error) {
if line[0] != '\\' || !bytes.HasSuffix(line, []byte("-grams:")) {
return nil, false, stream.ErrExpect(`section header "\N-grams:"`)
}
n, err := strconv.Atoi(string(line[1 : len(line)-len("-grams:")]))
if err != nil || n <= 0 {
return nil, false, stream.ErrExpect(`positive integer in section header "\N-grams:"`)
}
return newNgramEntries(n, it.builder), true, nil
}
func (it *ngramEntries) setParts(line []byte) error {
// p
x, xs := tokenSplit(line)
if x == "" {
return stream.ErrExpect("log-probability")
}
if f, err := strconv.ParseFloat(x, WEIGHT_SIZE); err != nil {
return err
} else {
it.p = Weight(f)
}
// context
for i := 1; i < it.n; i++ {
x, xs = tokenSplit(xs)
if x == "" {
return stream.ErrExpect(fmt.Sprintf("%d context word(s)", it.n))
}
it.context[i-1] = x
}
// word
x, xs = tokenSplit(xs)
if x == "" {
return stream.ErrExpect("word")
}
it.word = x
// bow
x, xs = tokenSplit(xs)
if x == "" {
it.bow = 0
} else if f, err := strconv.ParseFloat(x, WEIGHT_SIZE); err == nil {
it.bow = Weight(f)
} else {
return err
}
// no extra stuff
if len(xs) != 0 {
return stream.ErrExpect("end of line")
}
return nil
}
func (it ngramSection) Final() error { return stream.ErrExpect(`\N-grams: ...`) }