func TestTokenizer(t *testing.T) {
	logging.SetupTestLogging()

	for _, test := range tests {
		run_testcase(test, t)
	}
}
Ejemplo n.º 2
0
func TestTrecFileReader(t *testing.T) {
	logging.SetupTestLogging()

	log.Debugf("Creating FileReader")
	fr := new(TrecFileReader)

	log.Debugf("Opening file")
	fr.Init("test/testfile1.txt")

	log.Debugf("Reading file")
	doc := fr.Read()
	if id := doc.OrigIdent(); id != "12345" {
		t.Error("Failed to parse document id")
	}

	tokens := doc.Tokens()
	exp_tokens := expected()
	i := 0
	for tok := range tokens {
		exp, ok := <-exp_tokens
		if !ok {
			if tok.Type != NullToken {
				t.Error("Read all of 'expected' before reaching end of tokens")
			} else {

			}
		}
		i += 1

		if pos := tok.Position; pos != i {
			t.Errorf("'%s' was not at position %d as expected", tok, i)
		}

		if id := tok.DocId; id != doc.Identifier() {
			t.Errorf("Token '%s' did not have DocId matching '%s'", tok, id)
		}

		if tok.Text != exp {
			t.Errorf("%s did not match %s in position %d", tok, exp, i)
			break
		} else {
		}
	}
}