func TestTokenizer(t *testing.T) {
logging.SetupTestLogging()
for _, test := range tests {
run_testcase(test, t)
}
}
func TestTrecFileReader(t *testing.T) {
logging.SetupTestLogging()
log.Debugf("Creating FileReader")
fr := new(TrecFileReader)
log.Debugf("Opening file")
fr.Init("test/testfile1.txt")
log.Debugf("Reading file")
doc := fr.Read()
if id := doc.OrigIdent(); id != "12345" {
t.Error("Failed to parse document id")
}
tokens := doc.Tokens()
exp_tokens := expected()
i := 0
for tok := range tokens {
exp, ok := <-exp_tokens
if !ok {
if tok.Type != NullToken {
t.Error("Read all of 'expected' before reaching end of tokens")
} else {
}
}
i += 1
if pos := tok.Position; pos != i {
t.Errorf("'%s' was not at position %d as expected", tok, i)
}
if id := tok.DocId; id != doc.Identifier() {
t.Errorf("Token '%s' did not have DocId matching '%s'", tok, id)
}
if tok.Text != exp {
t.Errorf("%s did not match %s in position %d", tok, exp, i)
break
} else {
}
}
}