func TestTokenizer(t *testing.T) { logging.SetupTestLogging() for _, test := range tests { run_testcase(test, t) } }
func TestTrecFileReader(t *testing.T) { logging.SetupTestLogging() log.Debugf("Creating FileReader") fr := new(TrecFileReader) log.Debugf("Opening file") fr.Init("test/testfile1.txt") log.Debugf("Reading file") doc := fr.Read() if id := doc.OrigIdent(); id != "12345" { t.Error("Failed to parse document id") } tokens := doc.Tokens() exp_tokens := expected() i := 0 for tok := range tokens { exp, ok := <-exp_tokens if !ok { if tok.Type != NullToken { t.Error("Read all of 'expected' before reaching end of tokens") } else { } } i += 1 if pos := tok.Position; pos != i { t.Errorf("'%s' was not at position %d as expected", tok, i) } if id := tok.DocId; id != doc.Identifier() { t.Errorf("Token '%s' did not have DocId matching '%s'", tok, id) } if tok.Text != exp { t.Errorf("%s did not match %s in position %d", tok, exp, i) break } else { } } }