func NewTrainer(trainText string, fileText *os.File) *Trainer {
trainer := &Trainer{
Base: NewBase(),
TypeDist: utils.NewFreqDist(map[string]int{}),
CollocationDist: utils.NewFreqDist(map[string]int{}),
SentStarterDist: utils.NewFreqDist(map[string]int{}),
finalized: true,
Abbrev: 0.3,
AbbrevBackoff: 5,
Collocation: 7.88,
SentStarter: 30,
MinCollocFreq: 1,
IncludeAllCollocs: true,
}
if trainText != "" {
trainer.Train(trainText, true)
} else if fileText != nil {
reader := bufio.NewReader(fileText)
contents, _ := ioutil.ReadAll(reader)
trainer.Train(string(contents), true)
}
return trainer
}
func LoadFreqDist(data []byte) (*Trainer, error) {
var tn tmpDist
err := json.Unmarshal(data, &tn)
if err != nil {
return nil, err
}
trainer := NewTrainer("", nil)
trainer.TypeDist = utils.NewFreqDist(tn.TypeDist)
trainer.CollocationDist = utils.NewFreqDist(tn.CollocationDist)
trainer.SentStarterDist = utils.NewFreqDist(tn.SentStarterDist)
return trainer, nil
}