func BenchmarkAnalyze(b *testing.B) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
d := document.NewDocument("1")
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
d.AddField(f)
b.ResetTimer()
for i := 0; i < b.N; i++ {
rv := idx.Analyze(d)
if len(rv.Rows) < 92 || len(rv.Rows) > 93 {
b.Fatalf("expected 512-13 rows, got %d", len(rv.Rows))
}
}
}
func BenchmarkBatch(b *testing.B) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
err = idx.Open()
if err != nil {
b.Fatal(err)
}
batch := index.NewBatch()
for i := 0; i < 100; i++ {
d := document.NewDocument(strconv.Itoa(i))
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
d.AddField(f)
batch.Update(d)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
err = idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
}
func TestIndexTermReaderCompositeFields(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all")
if err != nil {
t.Error(err)
}
tfd, err := termFieldReader.Next()
for tfd != nil && err == nil {
if tfd.ID != "1" {
t.Errorf("expected to find document id 1")
}
tfd, err = termFieldReader.Next()
}
if err != nil {
t.Error(err)
}
}
func TestIndexInsertFields(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(dateField)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
fields, err := indexReader.Fields()
if err != nil {
t.Error(err)
} else {
expectedFields := []string{"name", "age", "unixEpoch"}
if !reflect.DeepEqual(fields, expectedFields) {
t.Errorf("expected fields: %v, got %v", expectedFields, fields)
}
}
}
func TestIndexInsert(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should have 4 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the term count, and 1 for the back index entry)
expectedLength := uint64(1 + 1 + 1 + 1 + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
}
func CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
err = idx.Open()
if err != nil {
b.Fatal(err)
}
b.StartTimer()
batch := index.NewBatch()
for j := 0; j < 1000; j++ {
if j%batchSize == 0 {
if len(batch.IndexOps) > 0 {
err := idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
batch = index.NewBatch()
}
indexDocument := document.NewDocument("").
AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j)
batch.Update(indexDocument)
}
// close last batch
if len(batch.IndexOps) > 0 {
err := idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
b.StopTimer()
err = idx.Close()
if err != nil {
b.Fatal(err)
}
err = destroy()
if err != nil {
b.Fatal(err)
}
analysisQueue.Close()
}
}
func TestBootstrap(t *testing.T) {
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
err = f.Open() // open calls bootstrap
if err != nil {
t.Fatal(err)
}
// assert that version is set
reader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
val, err := reader.Get(VersionKey)
if err != nil {
t.Fatal(err)
}
verRow, err := NewVersionRowV(val)
if err != nil {
t.Fatal(err)
}
if verRow.Version() != Version {
t.Errorf("expected version %d, got %d", Version, verRow.Version())
}
// assert that field cache has _id
id, existed := f.(*Firestorm).fieldCache.FieldNamed(IDFieldName, false)
if !existed {
t.Errorf("expect '%s' in field cache", IDFieldName)
}
if id != 0 {
t.Errorf("expected '%s' to have index 0, got %d", IDFieldName, id)
}
// assert that field is recorded in kv store
fRowExpected := NewFieldRow(id, IDFieldName)
fRowKey := fRowExpected.Key()
val, err = reader.Get(fRowKey)
if err != nil {
t.Fatal(err)
}
fRowActual, err := NewFieldRowKV(fRowKey, val)
if err != nil {
t.Fatal(err)
}
if fRowExpected.Name() != fRowActual.Name() {
t.Errorf("expected name '%s' got '%s'", fRowExpected.Name(), fRowActual.Name())
}
// assert that highDocNumber is 0
if f.(*Firestorm).highDocNumber != 0 {
t.Errorf("expected highDocNumber to be 0, got %d", f.(*Firestorm).highDocNumber)
}
}
func TestIndexOpenReopen(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// opening the database should have inserted a version
expectedLength := uint64(1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
// now close it
err = idx.Close()
if err != nil {
t.Fatal(err)
}
idx, err = NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
// now close it
err = idx.Close()
if err != nil {
t.Fatal(err)
}
}
func TestConcurrentUpdate(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
// do some concurrent updates
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func(i int) {
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), document.StoreField))
err := idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
wg.Done()
}(i)
}
wg.Wait()
// now load the name field and see what we get
r, err := idx.Reader()
if err != nil {
log.Fatal(err)
}
doc, err := r.Document("1")
if err != nil {
log.Fatal(err)
}
if len(doc.Fields) > 1 {
t.Errorf("expected single field, found %d", len(doc.Fields))
}
}
func TestIndexDocumentFieldTerms(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
fieldTerms, err := indexReader.DocumentFieldTerms("1")
if err != nil {
t.Error(err)
}
expectedFieldTerms := index.FieldTerms{
"name": []string{"test"},
"title": []string{"mister"},
}
if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) {
t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms)
}
}
func TestIndexInsert(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
if err != nil {
t.Fatal(err)
}
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
}
func TestIndexOpenReopen(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// now close it
err = idx.Close()
if err != nil {
t.Fatal(err)
}
idx, err = NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
// now close it
err = idx.Close()
if err != nil {
t.Fatal(err)
}
}
func TestIndexFieldsMisc(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
fieldName1 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(0)
if fieldName1 != "name" {
t.Errorf("expected field named 'name', got '%s'", fieldName1)
}
fieldName2 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(1)
if fieldName2 != "title" {
t.Errorf("expected field named 'title', got '%s'", fieldName2)
}
fieldName3 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(2)
if fieldName3 != "" {
t.Errorf("expected field named '', got '%s'", fieldName3)
}
}
func init() {
analysisQueue := index.NewAnalysisQueue(1)
var err error
twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
if err != nil {
panic(err)
}
err = twoDocIndex.Open()
if err != nil {
panic(err)
}
for _, doc := range twoDocIndexDocs {
err := twoDocIndex.Update(doc)
if err != nil {
panic(err)
}
}
}
func CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
if err != nil {
b.Fatal(err)
}
indexDocument := document.NewDocument("").
AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer))
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)
if err != nil {
b.Fatal(err)
}
err = idx.Open()
if err != nil {
b.Fatal(err)
}
indexDocument.ID = strconv.Itoa(i)
// just time the indexing portion
b.StartTimer()
err = idx.Update(indexDocument)
if err != nil {
b.Fatal(err)
}
b.StopTimer()
err = idx.Close()
if err != nil {
b.Fatal(err)
}
err = destroy()
if err != nil {
b.Fatal(err)
}
analysisQueue.Close()
}
}
func TestLargeField(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
largeFieldValue := make([]byte, 0)
for len(largeFieldValue) < RowBufferSize {
largeFieldValue = append(largeFieldValue, bleveWikiArticle1K...)
}
t.Logf("large field size: %d", len(largeFieldValue))
d := document.NewDocument("large")
f := document.NewTextFieldWithIndexingOptions("desc", nil, largeFieldValue, document.IndexField|document.StoreField)
d.AddField(f)
err = idx.Update(d)
if err != nil {
t.Fatal(err)
}
}
func TestIndexReader(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
doc = document.NewDocument("2")
doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
// first look for a term that doesn't exist
reader, err := indexReader.TermFieldReader([]byte("nope"), "name")
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
count := reader.Count()
if count != 0 {
t.Errorf("Expected doc count to be: %d got: %d", 0, count)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
reader, err = indexReader.TermFieldReader([]byte("test"), "name")
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
expectedCount = 2
count = reader.Count()
if count != expectedCount {
t.Errorf("Exptected doc count to be: %d got: %d", expectedCount, count)
}
var match *index.TermFieldDoc
var actualCount uint64
match, err = reader.Next()
for err == nil && match != nil {
match, err = reader.Next()
if err != nil {
t.Errorf("unexpected error reading next")
}
actualCount++
}
if actualCount != count {
t.Errorf("count was 2, but only saw %d", actualCount)
}
expectedMatch := &index.TermFieldDoc{
ID: "2",
Freq: 1,
Norm: 0.5773502588272095,
Vectors: []*index.TermFieldVector{
&index.TermFieldVector{
Field: "desc",
Pos: 3,
Start: 9,
End: 13,
},
},
}
tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
match, err = tfr.Next()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !reflect.DeepEqual(expectedMatch, match) {
t.Errorf("got %#v, expected %#v", match, expectedMatch)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// now test usage of advance
reader, err = indexReader.TermFieldReader([]byte("test"), "name")
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
match, err = reader.Advance("2")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if match == nil {
t.Fatalf("Expected match, got nil")
}
if match.ID != "2" {
t.Errorf("Expected ID '2', got '%s'", match.ID)
}
match, err = reader.Advance("3")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if match != nil {
t.Errorf("expected nil, got %v", match)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// now test creating a reader for a field that doesn't exist
reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist")
if err != nil {
t.Errorf("Error accessing term field reader: %v", err)
}
count = reader.Count()
if count != 0 {
t.Errorf("expected count 0 for reader of non-existant field")
}
match, err = reader.Next()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if match != nil {
t.Errorf("expected nil, got %v", match)
}
match, err = reader.Advance("anywhere")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if match != nil {
t.Errorf("expected nil, got %v", match)
}
}
func TestIndexDocIdReader(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test")))
doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Error(err)
}
}()
// first get all doc ids
reader, err := indexReader.DocIDReader("", "")
if err != nil {
t.Errorf("Error accessing doc id reader: %v", err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
id, err := reader.Next()
count := uint64(0)
for id != "" {
count++
id, err = reader.Next()
}
if count != expectedCount {
t.Errorf("expected %d, got %d", expectedCount, count)
}
// try it again, but jump to the second doc this time
reader2, err := indexReader.DocIDReader("", "")
if err != nil {
t.Errorf("Error accessing doc id reader: %v", err)
}
defer func() {
err := reader2.Close()
if err != nil {
t.Error(err)
}
}()
id, err = reader2.Advance("2")
if err != nil {
t.Error(err)
}
if id != "2" {
t.Errorf("expected to find id '2', got '%s'", id)
}
id, err = reader2.Advance("3")
if err != nil {
t.Error(err)
}
if id != "" {
t.Errorf("expected to find id '', got '%s'", id)
}
}
func newConfiguration() *configuration {
return &configuration{
Cache: registry.NewCache(),
analysisQueue: index.NewAnalysisQueue(4),
}
}
func TestIndexUpdateComposites(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
// should have 72 rows
// 1 for version
// 3 for schema fields
// 4 for text term
// 2 for the stored field
// 4 for the text term count
// 1 for the back index entry
expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
// now lets update it
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
// make sure new values are in index
storedDoc, err := indexReader.Document("1")
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 2 {
t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields))
}
textField, ok := storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "testupdated" {
t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
}
// should have the same row count as before, plus 4 term dictionary garbage rows
expectedLength += 4
rowCount, err = idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
}
func TestAnalysis(t *testing.T) {
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
err = f.Open()
if err != nil {
t.Fatal(err)
}
rows := []index.IndexRow{
NewFieldRow(0, IDFieldName),
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
for _, row := range rows {
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err := kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup to load field cache and set maxRead correctly
err = f.(*Firestorm).warmup(kvreader)
if err != nil {
t.Fatal(err)
}
tests := []struct {
d *document.Document
r *index.AnalysisResult
}{
{
d: document.NewDocument("a").
AddField(
document.NewTextFieldWithIndexingOptions("name", nil, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)),
r: &index.AnalysisResult{
DocID: "a",
Rows: []index.IndexRow{
NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil),
NewFieldRow(1, "name"),
NewStoredRow([]byte("a"), 1, 1, nil, []byte("ttest")),
NewTermFreqRow(1, []byte("test"), []byte("a"), 1, 1, 1.0, []*TermVector{NewTermVector(1, 1, 0, 4, nil)}),
},
},
},
}
for _, test := range tests {
test.d.Number = 1
actual := f.Analyze(test.d)
if !reflect.DeepEqual(actual, test.r) {
t.Errorf("expected: %v got %v", test.r, actual)
}
}
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
}
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(df)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should have 72 rows
// 1 for version
// 3 for schema fields
// 1 for text term
// 16 for numeric terms
// 16 for date terms
// 3 for the stored field
// 1 for the text term count
// 16 for numeric term counts
// 16 for date term counts
// 1 for the back index entry
expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
storedDoc, err := indexReader.Document("1")
if err != nil {
t.Error(err)
}
err = indexReader.Close()
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 3 {
t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields))
}
textField, ok := storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "test" {
t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
}
numField, ok := storedDoc.Fields[1].(*document.NumericField)
if !ok {
t.Errorf("expected numeric field")
}
numFieldNumer, err := numField.Number()
if err != nil {
t.Error(err)
} else {
if numFieldNumer != 35.99 {
t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer)
}
}
dateField, ok := storedDoc.Fields[2].(*document.DateTimeField)
if !ok {
t.Errorf("expected date field")
}
dateFieldDate, err := dateField.DateTime()
if err != nil {
t.Error(err)
} else {
if dateFieldDate != time.Unix(0, 0).UTC() {
t.Errorf("expected date value unix epoch, got %v", dateFieldDate)
}
}
// now update the document, but omit one of the fields
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader2, err := idx.Reader()
if err != nil {
t.Error(err)
}
// expected doc count shouldn't have changed
docCount = indexReader2.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should only get 2 fields back now though
storedDoc, err = indexReader2.Document("1")
if err != nil {
t.Error(err)
}
err = indexReader2.Close()
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 2 {
t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields))
}
textField, ok = storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "testup" {
t.Errorf("expected field content 'testup', got '%s'", string(textField.Value()))
}
numField, ok = storedDoc.Fields[1].(*document.NumericField)
if !ok {
t.Errorf("expected numeric field")
}
numFieldNumer, err = numField.Number()
if err != nil {
t.Error(err)
} else {
if numFieldNumer != 36.99 {
t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer)
}
}
// now delete the document
err = idx.Delete("1")
expectedCount--
// expected doc count shouldn't have changed
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
}
func TestIndexInsertThenUpdate(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
// this update should overwrite one term, and introduce one new one
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error deleting entry from index: %v", err)
}
// should have 2 rows (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry)
expectedLength := uint64(1 + 1 + 2 + 2 + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
// now do another update that should remove one of the terms
doc = document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error deleting entry from index: %v", err)
}
// should have 2 rows (1 for version, 1 for schema field, and 1 for the remaining term, and 2 for the term diciontary, and 1 for the back index entry)
expectedLength = uint64(1 + 1 + 1 + 2 + 1)
rowCount, err = idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
}
func TestDocIDReaderSomeGarbage(t *testing.T) {
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
rows := []index.IndexRow{
NewFieldRow(0, IDFieldName),
NewFieldRow(1, "desc"),
NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("b"), 2, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("c"), 3, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("d"), 4, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("a"), 5, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("b"), 6, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("e"), 7, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("g"), 8, 0, 0.0, nil),
// first version of all docs have cat
NewTermFreqRow(1, []byte("cat"), []byte("a"), 1, 1, 1.0, nil),
NewTermFreqRow(1, []byte("cat"), []byte("b"), 2, 1, 1.0, nil),
NewTermFreqRow(1, []byte("cat"), []byte("c"), 3, 1, 1.0, nil),
NewTermFreqRow(1, []byte("cat"), []byte("d"), 4, 1, 1.0, nil),
NewTermFreqRow(1, []byte("cat"), []byte("e"), 7, 1, 1.0, nil),
NewTermFreqRow(1, []byte("cat"), []byte("g"), 8, 1, 1.0, nil),
// updated version of a still has cat
NewTermFreqRow(1, []byte("cat"), []byte("a"), 5, 1, 1.0, nil),
// updated version of b does NOT have cat
// c has delete in-flight
// d has delete not-yet-garbage-collected
}
for _, row := range rows {
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
f.(*Firestorm).compensator.inFlight = f.(*Firestorm).compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int())
f.(*Firestorm).compensator.deletedDocNumbers.Set(4)
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
// warmup to load field cache and set maxRead correctly
err = f.(*Firestorm).warmup(kvreader)
if err != nil {
t.Fatal(err)
}
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
r, err := f.Reader()
if err != nil {
t.Fatal(err)
}
dr, err := r.DocIDReader("", "")
if err != nil {
t.Fatal(err)
}
expectedDocIds := []string{"a", "b", "e", "g"}
foundDocIds := make([]string, 0)
next, err := dr.Next()
for next != "" && err == nil {
foundDocIds = append(foundDocIds, next)
next, err = dr.Next()
}
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(expectedDocIds, foundDocIds) {
t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds)
}
err = dr.Close()
if err != nil {
t.Fatal(err)
}
// now test with some doc id ranges
dr, err = r.DocIDReader("b", "f")
if err != nil {
t.Fatal(err)
}
expectedDocIds = []string{"b", "e"}
foundDocIds = make([]string, 0)
next, err = dr.Next()
for next != "" && err == nil {
foundDocIds = append(foundDocIds, next)
next, err = dr.Next()
}
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(expectedDocIds, foundDocIds) {
t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds)
}
err = dr.Close()
if err != nil {
t.Fatal(err)
}
//now try again and Advance to skip over "e"
dr, err = r.DocIDReader("b", "")
if err != nil {
t.Fatal(err)
}
expectedDocIds = []string{"b", "g"}
foundDocIds = make([]string, 0)
next, err = dr.Next()
if err != nil {
t.Fatal(err)
} else {
foundDocIds = append(foundDocIds, next)
}
next, err = dr.Advance("f")
if err != nil {
t.Fatal(err)
} else {
foundDocIds = append(foundDocIds, next)
}
if !reflect.DeepEqual(expectedDocIds, foundDocIds) {
t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds)
}
err = dr.Close()
if err != nil {
t.Fatal(err)
}
err = r.Close()
if err != nil {
t.Fatal(err)
}
}
func TestTermSearcher(t *testing.T) {
var queryTerm = "beer"
var queryField = "desc"
var queryBoost = 3.0
var queryExplain = true
analysisQueue := index.NewAnalysisQueue(1)
i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = i.Open()
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "a",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "b",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "c",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "d",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "e",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "f",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "g",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "h",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "i",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "j",
Fields: []document.Field{
document.NewTextField("title", []uint64{}, []byte("cat")),
},
})
if err != nil {
t.Fatal(err)
}
indexReader, err := i.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain)
if err != nil {
t.Fatal(err)
}
defer func() {
err := searcher.Close()
if err != nil {
t.Fatal(err)
}
}()
searcher.SetQueryNorm(2.0)
docCount, err := i.DocCount()
if err != nil {
t.Fatal(err)
}
idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0))
expectedQueryWeight := 3 * idf * 3 * idf
if expectedQueryWeight != searcher.Weight() {
t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight())
}
if searcher.Count() != 9 {
t.Errorf("expected count of 9, got %d", searcher.Count())
}
docMatch, err := searcher.Next()
if err != nil {
t.Errorf("expected result, got %v", err)
}
if docMatch.ID != "a" {
t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID)
}
docMatch, err = searcher.Advance("c")
if err != nil {
t.Errorf("expected result, got %v", err)
}
if docMatch.ID != "c" {
t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID)
}
// try advancing past end
docMatch, err = searcher.Advance("z")
if err != nil {
t.Fatal(err)
}
if docMatch != nil {
t.Errorf("expected nil, got %v", docMatch)
}
// try pushing next past end
docMatch, err = searcher.Next()
if err != nil {
t.Fatal(err)
}
if docMatch != nil {
t.Errorf("expected nil, got %v", docMatch)
}
}
func TestIndexInsertWithStore(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count, and 1 for the back index entry)
expectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
storedDoc, err := indexReader.Document("1")
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 1 {
t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields))
}
textField, ok := storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "test" {
t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
}
}
func TestLookups(t *testing.T) {
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
rows := []struct {
row index.IndexRow
garbage bool
}{
// needed for warmup to work
{NewFieldRow(0, IDFieldName), false},
// 3 documents, with 2 older versions
{NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil), true},
{NewTermFreqRow(0, nil, []byte("a"), 2, 0, 0.0, nil), false},
{NewTermFreqRow(0, nil, []byte("b"), 3, 0, 0.0, nil), false},
{NewTermFreqRow(0, nil, []byte("c"), 4, 0, 0.0, nil), true},
{NewTermFreqRow(0, nil, []byte("c"), 5, 0, 0.0, nil), false},
}
for _, row := range rows {
wb := kvwriter.NewBatch()
wb.Set(row.row.Key(), row.row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
// also see the compensator
if tfr, ok := row.row.(*TermFreqRow); ok {
f.(*Firestorm).compensator.Mutate(tfr.DocID(), tfr.DocNum())
// expect this mutation to be in the in-flight list
val := f.(*Firestorm).compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()})
if val == nil {
t.Errorf("expected key: % x to be in the inflight list", tfr.DocID())
}
f.(*Firestorm).lookuper.lookup(&InFlightItem{docID: tfr.DocID(), docNum: tfr.DocNum()})
// now expect this mutation to NOT be in the in-flight list
val = f.(*Firestorm).compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()})
if val != nil {
t.Errorf("expected key: % x to NOT be in the inflight list, got %v", tfr.DocID(), val)
}
}
}
// check that doc count is 3 at the end of this
docCount, err := f.DocCount()
if err != nil {
t.Fatal(err)
}
if docCount != 3 {
t.Errorf("expected doc count 3, got %d", docCount)
}
}
func TestIndexInternalCRUD(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
// get something that doesn't exist yet
val, err := indexReader.GetInternal([]byte("key"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %s", val)
}
err = indexReader.Close()
if err != nil {
t.Fatal(err)
}
// set
err = idx.SetInternal([]byte("key"), []byte("abc"))
if err != nil {
t.Error(err)
}
indexReader2, err := idx.Reader()
if err != nil {
t.Error(err)
}
// get
val, err = indexReader2.GetInternal([]byte("key"))
if err != nil {
t.Error(err)
}
if string(val) != "abc" {
t.Errorf("expected %s, got '%s'", "abc", val)
}
err = indexReader2.Close()
if err != nil {
t.Fatal(err)
}
// delete
err = idx.DeleteInternal([]byte("key"))
if err != nil {
t.Error(err)
}
indexReader3, err := idx.Reader()
if err != nil {
t.Error(err)
}
// get again
val, err = indexReader3.GetInternal([]byte("key"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %s", val)
}
err = indexReader3.Close()
if err != nil {
t.Fatal(err)
}
}
func TestWarmupNoGarbage(t *testing.T) {
aq := index.NewAnalysisQueue(1)
f, err := NewFirestorm(gtreap.Name, nil, aq)
if err != nil {
t.Fatal(err)
}
err = f.Open()
if err != nil {
t.Fatal(err)
}
kvwriter, err := f.(*Firestorm).store.Writer()
if err != nil {
t.Fatal(err)
}
rows := []index.IndexRow{
NewFieldRow(0, IDFieldName),
NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("b"), 2, 0, 0.0, nil),
NewTermFreqRow(0, nil, []byte("c"), 3, 0, 0.0, nil),
}
expectedCount := uint64(3)
expectedGarbage := uint64(0)
for _, row := range rows {
wb := kvwriter.NewBatch()
wb.Set(row.Key(), row.Value())
err = kvwriter.ExecuteBatch(wb)
if err != nil {
t.Fatal(err)
}
}
err = kvwriter.Close()
if err != nil {
t.Fatal(err)
}
kvreader, err := f.(*Firestorm).store.Reader()
if err != nil {
t.Fatal(err)
}
err = f.(*Firestorm).warmup(kvreader)
if err != nil {
t.Fatal(err)
}
err = kvreader.Close()
if err != nil {
t.Fatal(err)
}
// assert that doc count is correct
count, err := f.DocCount()
if err != nil {
t.Fatal(err)
}
if count != expectedCount {
t.Errorf("expected doc count %d, got %d", expectedCount, count)
}
// assert that deleted doc numbers size is 0
if f.(*Firestorm).compensator.GarbageCount() != expectedGarbage {
t.Errorf("expected 0 deleted doc numbers, got %d", f.(*Firestorm).compensator.GarbageCount())
}
// assert that highDocNumber is 3
if f.(*Firestorm).highDocNumber != 3 {
t.Errorf("expected highDocNumber to be 3, got %d", f.(*Firestorm).highDocNumber)
}
}
func TestIndexBatch(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
// first create 2 docs the old fashioned way
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
// now create a batch which does 3 things
// insert new doc
// update existing doc
// delete existing doc
// net document count change 0
batch := index.NewBatch()
doc = document.NewDocument("3")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3")))
batch.Update(doc)
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated")))
batch.Update(doc)
batch.Delete("1")
err = idx.Batch(batch)
if err != nil {
t.Error(err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
docCount := indexReader.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
docIDReader, err := indexReader.DocIDReader("", "")
if err != nil {
t.Error(err)
}
docIds := make([]string, 0)
docID, err := docIDReader.Next()
for docID != "" && err == nil {
docIds = append(docIds, docID)
docID, err = docIDReader.Next()
}
if err != nil {
t.Error(err)
}
expectedDocIds := []string{"2", "3"}
if !reflect.DeepEqual(docIds, expectedDocIds) {
t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds)
}
}