func readInstances(reader io.Reader, f extractFunc) (map[string]interface{}, error) {
conllReader := conllx.NewReader(bufio.NewReader(reader))
vals := make(map[string]interface{})
for {
s, err := conllReader.ReadSentence()
if err != nil {
if err != io.EOF {
exitIfErr(err)
}
break
}
for _, token := range s {
tokenVals, avail := f(token)
if !avail {
//exitIfErr(errors.New("Token without relevant layer encountered"))
continue
}
for _, val := range tokenVals {
vals[val] = nil
}
}
}
return vals, nil
}
func run(parser system.Parser) {
inputFile, err := os.Open(flag.Arg(1))
defer inputFile.Close()
if err != nil {
panic("Cannot open training data")
}
inputReader := conllx.NewReader(bufio.NewReader(inputFile))
writer := conllx.NewWriter(os.Stdout)
for {
s, err := inputReader.ReadSentence()
if err != nil {
break
}
deps, err := parser.Parse(s)
common.ExitIfError(err)
// Clear to ensure that no dependencies in the input leak
// (if they were present).
for idx := range s {
s[idx].SetHead(0)
s[idx].SetHeadRel("NULL")
}
for dep := range deps {
s[dep.Dependent-1].SetHead(dep.Head)
s[dep.Dependent-1].SetHeadRel(dep.Relation)
}
writer.WriteSentence(s)
}
}
func evaluate(parser system.Parser) {
testFile, err := os.Open(flag.Arg(1))
defer testFile.Close()
if err != nil {
panic("Cannot open evaluation data")
}
testReader := conllx.NewReader(bufio.NewReader(testFile))
total := 0
found := 0
count := 0
for {
s, err := testReader.ReadSentence()
if err != nil {
break
}
goldDeps, err := system.SentenceToDependencies(s)
if err != nil {
log.Fatal(err)
}
deps, err := parser.Parse(s)
if err != nil {
log.Fatal(err)
}
total += len(goldDeps)
found += foundAttachments(goldDeps, deps)
count++
if count%100 == 0 {
printAccuracy(found, total)
}
}
printAccuracy(found, total)
}
func ProcessData(f *os.File, fun DataFun) error {
bar, err := CreateFileProgress(f)
if err != nil {
return err
}
defer bar.Finish()
bar.Start()
r := conllx.NewReader(bufio.NewReader(bar.NewProxyReader(f)))
for {
s, err := r.ReadSentence()
if err != nil {
break
}
if err := fun(s); err != nil {
return err
}
}
return nil
}
func createTrainingInstances(trainer system.GreedyTrainer, collector system.InstanceCollector,
oracleConstructor common.OracleConstructor) {
f, err := os.Open(flag.Arg(1))
defer f.Close()
if err != nil {
panic("Cannot open training data")
}
r := conllx.NewReader(bufio.NewReader(f))
for {
s, err := r.ReadSentence()
if err != nil {
break
}
goldDependencies, err := system.SentenceToDependencies(s)
common.ExitIfError(err)
oracle := oracleConstructor(goldDependencies)
trainer.Parse(s, oracle)
}
}