func BenchmarkAnalyze(b *testing.B) { cache := registry.NewCache() analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name) if err != nil { b.Fatal(err) } analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(null.Name, nil, analysisQueue) if err != nil { b.Fatal(err) } d := document.NewDocument("1") f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer) d.AddField(f) b.ResetTimer() for i := 0; i < b.N; i++ { rv := idx.Analyze(d) if len(rv.Rows) < 92 || len(rv.Rows) > 93 { b.Fatalf("expected 512-13 rows, got %d", len(rv.Rows)) } } }
func BenchmarkBatch(b *testing.B) { cache := registry.NewCache() analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name) if err != nil { b.Fatal(err) } analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue) if err != nil { b.Fatal(err) } err = idx.Open() if err != nil { b.Fatal(err) } batch := index.NewBatch() for i := 0; i < 100; i++ { d := document.NewDocument(strconv.Itoa(i)) f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer) d.AddField(f) batch.Update(d) } b.ResetTimer() for i := 0; i < b.N; i++ { err = idx.Batch(batch) if err != nil { b.Fatal(err) } } }
func TestIndexTermReaderCompositeFields(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") if err != nil { t.Error(err) } tfd, err := termFieldReader.Next() for tfd != nil && err == nil { if tfd.ID != "1" { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next() } if err != nil { t.Error(err) } }
func TestIndexInsertFields(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() fields, err := indexReader.Fields() if err != nil { t.Error(err) } else { expectedFields := []string{"name", "age", "unixEpoch"} if !reflect.DeepEqual(fields, expectedFields) { t.Errorf("expected fields: %v, got %v", expectedFields, fields) } } }
func TestIndexInsert(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should have 4 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the term count, and 1 for the back index entry) expectedLength := uint64(1 + 1 + 1 + 1 + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } }
func CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) { cache := registry.NewCache() analyzer, err := cache.AnalyzerNamed("standard") if err != nil { b.Fatal(err) } b.ResetTimer() b.StopTimer() for i := 0; i < b.N; i++ { analysisQueue := index.NewAnalysisQueue(analysisWorkers) idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue) if err != nil { b.Fatal(err) } err = idx.Open() if err != nil { b.Fatal(err) } b.StartTimer() batch := index.NewBatch() for j := 0; j < 1000; j++ { if j%batchSize == 0 { if len(batch.IndexOps) > 0 { err := idx.Batch(batch) if err != nil { b.Fatal(err) } } batch = index.NewBatch() } indexDocument := document.NewDocument(""). AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer)) indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j) batch.Update(indexDocument) } // close last batch if len(batch.IndexOps) > 0 { err := idx.Batch(batch) if err != nil { b.Fatal(err) } } b.StopTimer() err = idx.Close() if err != nil { b.Fatal(err) } err = destroy() if err != nil { b.Fatal(err) } analysisQueue.Close() } }
func TestBootstrap(t *testing.T) { aq := index.NewAnalysisQueue(1) f, err := NewFirestorm(gtreap.Name, nil, aq) if err != nil { t.Fatal(err) } err = f.Open() // open calls bootstrap if err != nil { t.Fatal(err) } // assert that version is set reader, err := f.(*Firestorm).store.Reader() if err != nil { t.Fatal(err) } val, err := reader.Get(VersionKey) if err != nil { t.Fatal(err) } verRow, err := NewVersionRowV(val) if err != nil { t.Fatal(err) } if verRow.Version() != Version { t.Errorf("expected version %d, got %d", Version, verRow.Version()) } // assert that field cache has _id id, existed := f.(*Firestorm).fieldCache.FieldNamed(IDFieldName, false) if !existed { t.Errorf("expect '%s' in field cache", IDFieldName) } if id != 0 { t.Errorf("expected '%s' to have index 0, got %d", IDFieldName, id) } // assert that field is recorded in kv store fRowExpected := NewFieldRow(id, IDFieldName) fRowKey := fRowExpected.Key() val, err = reader.Get(fRowKey) if err != nil { t.Fatal(err) } fRowActual, err := NewFieldRowKV(fRowKey, val) if err != nil { t.Fatal(err) } if fRowExpected.Name() != fRowActual.Name() { t.Errorf("expected name '%s' got '%s'", fRowExpected.Name(), fRowActual.Name()) } // assert that highDocNumber is 0 if f.(*Firestorm).highDocNumber != 0 { t.Errorf("expected highDocNumber to be 0, got %d", f.(*Firestorm).highDocNumber) } }
func TestIndexOpenReopen(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // opening the database should have inserted a version expectedLength := uint64(1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } // now close it err = idx.Close() if err != nil { t.Fatal(err) } idx, err = NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } // now close it err = idx.Close() if err != nil { t.Fatal(err) } }
func TestConcurrentUpdate(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() // do some concurrent updates var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) go func(i int) { doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), document.StoreField)) err := idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } wg.Done() }(i) } wg.Wait() // now load the name field and see what we get r, err := idx.Reader() if err != nil { log.Fatal(err) } doc, err := r.Document("1") if err != nil { log.Fatal(err) } if len(doc.Fields) > 1 { t.Errorf("expected single field, found %d", len(doc.Fields)) } }
func TestIndexDocumentFieldTerms(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() fieldTerms, err := indexReader.DocumentFieldTerms("1") if err != nil { t.Error(err) } expectedFieldTerms := index.FieldTerms{ "name": []string{"test"}, "title": []string{"mister"}, } if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) { t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms) } }
func TestIndexInsert(t *testing.T) { defer func() { err := os.RemoveAll("test") if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration) if err != nil { t.Fatal(err) } docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } }
func TestIndexOpenReopen(t *testing.T) { defer func() { err := os.RemoveAll("test") if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // now close it err = idx.Close() if err != nil { t.Fatal(err) } idx, err = NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } // now close it err = idx.Close() if err != nil { t.Fatal(err) } }
func TestIndexFieldsMisc(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } fieldName1 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(0) if fieldName1 != "name" { t.Errorf("expected field named 'name', got '%s'", fieldName1) } fieldName2 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(1) if fieldName2 != "title" { t.Errorf("expected field named 'title', got '%s'", fieldName2) } fieldName3 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(2) if fieldName3 != "" { t.Errorf("expected field named '', got '%s'", fieldName3) } }
func init() { analysisQueue := index.NewAnalysisQueue(1) var err error twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { panic(err) } err = twoDocIndex.Open() if err != nil { panic(err) } for _, doc := range twoDocIndexDocs { err := twoDocIndex.Update(doc) if err != nil { panic(err) } } }
func CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) { cache := registry.NewCache() analyzer, err := cache.AnalyzerNamed("standard") if err != nil { b.Fatal(err) } indexDocument := document.NewDocument(""). AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer)) b.ResetTimer() b.StopTimer() for i := 0; i < b.N; i++ { analysisQueue := index.NewAnalysisQueue(analysisWorkers) idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue) if err != nil { b.Fatal(err) } err = idx.Open() if err != nil { b.Fatal(err) } indexDocument.ID = strconv.Itoa(i) // just time the indexing portion b.StartTimer() err = idx.Update(indexDocument) if err != nil { b.Fatal(err) } b.StopTimer() err = idx.Close() if err != nil { b.Fatal(err) } err = destroy() if err != nil { b.Fatal(err) } analysisQueue.Close() } }
func TestLargeField(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() largeFieldValue := make([]byte, 0) for len(largeFieldValue) < RowBufferSize { largeFieldValue = append(largeFieldValue, bleveWikiArticle1K...) } t.Logf("large field size: %d", len(largeFieldValue)) d := document.NewDocument("large") f := document.NewTextFieldWithIndexingOptions("desc", nil, largeFieldValue, document.IndexField|document.StoreField) d.AddField(f) err = idx.Update(d) if err != nil { t.Fatal(err) } }
func TestIndexReader(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 doc := document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ doc = document.NewDocument("2") doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() // first look for a term that doesn't exist reader, err := indexReader.TermFieldReader([]byte("nope"), "name") if err != nil { t.Errorf("Error accessing term field reader: %v", err) } count := reader.Count() if count != 0 { t.Errorf("Expected doc count to be: %d got: %d", 0, count) } err = reader.Close() if err != nil { t.Fatal(err) } reader, err = indexReader.TermFieldReader([]byte("test"), "name") if err != nil { t.Errorf("Error accessing term field reader: %v", err) } expectedCount = 2 count = reader.Count() if count != expectedCount { t.Errorf("Exptected doc count to be: %d got: %d", expectedCount, count) } var match *index.TermFieldDoc var actualCount uint64 match, err = reader.Next() for err == nil && match != nil { match, err = reader.Next() if err != nil { t.Errorf("unexpected error reading next") } actualCount++ } if actualCount != count { t.Errorf("count was 2, but only saw %d", actualCount) } expectedMatch := &index.TermFieldDoc{ ID: "2", Freq: 1, Norm: 0.5773502588272095, Vectors: []*index.TermFieldVector{ &index.TermFieldVector{ Field: "desc", Pos: 3, Start: 9, End: 13, }, }, } tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc") if err != nil { t.Errorf("unexpected error: %v", err) } match, err = tfr.Next() if err != nil { t.Errorf("unexpected error: %v", err) } if !reflect.DeepEqual(expectedMatch, match) { t.Errorf("got %#v, expected %#v", match, expectedMatch) } err = reader.Close() if err != nil { t.Fatal(err) } // now test usage of advance reader, err = indexReader.TermFieldReader([]byte("test"), "name") if err != nil { t.Errorf("Error accessing term field reader: %v", err) } match, err = reader.Advance("2") if err != nil { t.Errorf("unexpected error: %v", err) } if match == nil { t.Fatalf("Expected match, got nil") } if match.ID != "2" { t.Errorf("Expected ID '2', got '%s'", match.ID) } match, err = reader.Advance("3") if err != nil { t.Errorf("unexpected error: %v", err) } if match != nil { t.Errorf("expected nil, got %v", match) } err = reader.Close() if err != nil { t.Fatal(err) } // now test creating a reader for a field that doesn't exist reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist") if err != nil { t.Errorf("Error accessing term field reader: %v", err) } count = reader.Count() if count != 0 { t.Errorf("expected count 0 for reader of non-existant field") } match, err = reader.Next() if err != nil { t.Errorf("unexpected error: %v", err) } if match != nil { t.Errorf("expected nil, got %v", match) } match, err = reader.Advance("anywhere") if err != nil { t.Errorf("unexpected error: %v", err) } if match != nil { t.Errorf("expected nil, got %v", match) } }
func TestIndexDocIdReader(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 doc := document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ doc = document.NewDocument("2") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test"))) doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Error(err) } }() // first get all doc ids reader, err := indexReader.DocIDReader("", "") if err != nil { t.Errorf("Error accessing doc id reader: %v", err) } defer func() { err := reader.Close() if err != nil { t.Fatal(err) } }() id, err := reader.Next() count := uint64(0) for id != "" { count++ id, err = reader.Next() } if count != expectedCount { t.Errorf("expected %d, got %d", expectedCount, count) } // try it again, but jump to the second doc this time reader2, err := indexReader.DocIDReader("", "") if err != nil { t.Errorf("Error accessing doc id reader: %v", err) } defer func() { err := reader2.Close() if err != nil { t.Error(err) } }() id, err = reader2.Advance("2") if err != nil { t.Error(err) } if id != "2" { t.Errorf("expected to find id '2', got '%s'", id) } id, err = reader2.Advance("3") if err != nil { t.Error(err) } if id != "" { t.Errorf("expected to find id '', got '%s'", id) } }
func newConfiguration() *configuration { return &configuration{ Cache: registry.NewCache(), analysisQueue: index.NewAnalysisQueue(4), } }
func TestIndexUpdateComposites(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } // should have 72 rows // 1 for version // 3 for schema fields // 4 for text term // 2 for the stored field // 4 for the text term count // 1 for the back index entry expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } // now lets update it doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() // make sure new values are in index storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testupdated" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } // should have the same row count as before, plus 4 term dictionary garbage rows expectedLength += 4 rowCount, err = idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } }
func TestAnalysis(t *testing.T) { aq := index.NewAnalysisQueue(1) f, err := NewFirestorm(gtreap.Name, nil, aq) if err != nil { t.Fatal(err) } err = f.Open() if err != nil { t.Fatal(err) } rows := []index.IndexRow{ NewFieldRow(0, IDFieldName), } kvwriter, err := f.(*Firestorm).store.Writer() if err != nil { t.Fatal(err) } for _, row := range rows { wb := kvwriter.NewBatch() wb.Set(row.Key(), row.Value()) err := kvwriter.ExecuteBatch(wb) if err != nil { t.Fatal(err) } } err = kvwriter.Close() if err != nil { t.Fatal(err) } kvreader, err := f.(*Firestorm).store.Reader() if err != nil { t.Fatal(err) } // warmup to load field cache and set maxRead correctly err = f.(*Firestorm).warmup(kvreader) if err != nil { t.Fatal(err) } tests := []struct { d *document.Document r *index.AnalysisResult }{ { d: document.NewDocument("a"). AddField( document.NewTextFieldWithIndexingOptions("name", nil, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)), r: &index.AnalysisResult{ DocID: "a", Rows: []index.IndexRow{ NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil), NewFieldRow(1, "name"), NewStoredRow([]byte("a"), 1, 1, nil, []byte("ttest")), NewTermFreqRow(1, []byte("test"), []byte("a"), 1, 1, 1.0, []*TermVector{NewTermVector(1, 1, 0, 4, nil)}), }, }, }, } for _, test := range tests { test.d.Number = 1 actual := f.Analyze(test.d) if !reflect.DeepEqual(actual, test.r) { t.Errorf("expected: %v got %v", test.r, actual) } } err = kvreader.Close() if err != nil { t.Fatal(err) } }
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(df) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should have 72 rows // 1 for version // 3 for schema fields // 1 for text term // 16 for numeric terms // 16 for date terms // 3 for the stored field // 1 for the text term count // 16 for numeric term counts // 16 for date term counts // 1 for the back index entry expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } err = indexReader.Close() if err != nil { t.Error(err) } if len(storedDoc.Fields) != 3 { t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "test" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } numField, ok := storedDoc.Fields[1].(*document.NumericField) if !ok { t.Errorf("expected numeric field") } numFieldNumer, err := numField.Number() if err != nil { t.Error(err) } else { if numFieldNumer != 35.99 { t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer) } } dateField, ok := storedDoc.Fields[2].(*document.DateTimeField) if !ok { t.Errorf("expected date field") } dateFieldDate, err := dateField.DateTime() if err != nil { t.Error(err) } else { if dateFieldDate != time.Unix(0, 0).UTC() { t.Errorf("expected date value unix epoch, got %v", dateFieldDate) } } // now update the document, but omit one of the fields doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader2, err := idx.Reader() if err != nil { t.Error(err) } // expected doc count shouldn't have changed docCount = indexReader2.DocCount() if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should only get 2 fields back now though storedDoc, err = indexReader2.Document("1") if err != nil { t.Error(err) } err = indexReader2.Close() if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) } textField, ok = storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testup" { t.Errorf("expected field content 'testup', got '%s'", string(textField.Value())) } numField, ok = storedDoc.Fields[1].(*document.NumericField) if !ok { t.Errorf("expected numeric field") } numFieldNumer, err = numField.Number() if err != nil { t.Error(err) } else { if numFieldNumer != 36.99 { t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer) } } // now delete the document err = idx.Delete("1") expectedCount-- // expected doc count shouldn't have changed docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } }
func TestIndexInsertThenUpdate(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } // this update should overwrite one term, and introduce one new one doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer)) err = idx.Update(doc) if err != nil { t.Errorf("Error deleting entry from index: %v", err) } // should have 2 rows (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry) expectedLength := uint64(1 + 1 + 2 + 2 + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } // now do another update that should remove one of the terms doc = document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail"))) err = idx.Update(doc) if err != nil { t.Errorf("Error deleting entry from index: %v", err) } // should have 2 rows (1 for version, 1 for schema field, and 1 for the remaining term, and 2 for the term diciontary, and 1 for the back index entry) expectedLength = uint64(1 + 1 + 1 + 2 + 1) rowCount, err = idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } }
func TestDocIDReaderSomeGarbage(t *testing.T) { aq := index.NewAnalysisQueue(1) f, err := NewFirestorm(gtreap.Name, nil, aq) if err != nil { t.Fatal(err) } err = f.Open() if err != nil { t.Fatal(err) } kvwriter, err := f.(*Firestorm).store.Writer() if err != nil { t.Fatal(err) } rows := []index.IndexRow{ NewFieldRow(0, IDFieldName), NewFieldRow(1, "desc"), NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("b"), 2, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("c"), 3, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("d"), 4, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("a"), 5, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("b"), 6, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("e"), 7, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("g"), 8, 0, 0.0, nil), // first version of all docs have cat NewTermFreqRow(1, []byte("cat"), []byte("a"), 1, 1, 1.0, nil), NewTermFreqRow(1, []byte("cat"), []byte("b"), 2, 1, 1.0, nil), NewTermFreqRow(1, []byte("cat"), []byte("c"), 3, 1, 1.0, nil), NewTermFreqRow(1, []byte("cat"), []byte("d"), 4, 1, 1.0, nil), NewTermFreqRow(1, []byte("cat"), []byte("e"), 7, 1, 1.0, nil), NewTermFreqRow(1, []byte("cat"), []byte("g"), 8, 1, 1.0, nil), // updated version of a still has cat NewTermFreqRow(1, []byte("cat"), []byte("a"), 5, 1, 1.0, nil), // updated version of b does NOT have cat // c has delete in-flight // d has delete not-yet-garbage-collected } for _, row := range rows { wb := kvwriter.NewBatch() wb.Set(row.Key(), row.Value()) err = kvwriter.ExecuteBatch(wb) if err != nil { t.Fatal(err) } } f.(*Firestorm).compensator.inFlight = f.(*Firestorm).compensator.inFlight.Upsert(&InFlightItem{docID: []byte("c"), docNum: 0}, rand.Int()) f.(*Firestorm).compensator.deletedDocNumbers.Set(4) err = kvwriter.Close() if err != nil { t.Fatal(err) } kvreader, err := f.(*Firestorm).store.Reader() if err != nil { t.Fatal(err) } // warmup to load field cache and set maxRead correctly err = f.(*Firestorm).warmup(kvreader) if err != nil { t.Fatal(err) } err = kvreader.Close() if err != nil { t.Fatal(err) } r, err := f.Reader() if err != nil { t.Fatal(err) } dr, err := r.DocIDReader("", "") if err != nil { t.Fatal(err) } expectedDocIds := []string{"a", "b", "e", "g"} foundDocIds := make([]string, 0) next, err := dr.Next() for next != "" && err == nil { foundDocIds = append(foundDocIds, next) next, err = dr.Next() } if err != nil { t.Fatal(err) } if !reflect.DeepEqual(expectedDocIds, foundDocIds) { t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds) } err = dr.Close() if err != nil { t.Fatal(err) } // now test with some doc id ranges dr, err = r.DocIDReader("b", "f") if err != nil { t.Fatal(err) } expectedDocIds = []string{"b", "e"} foundDocIds = make([]string, 0) next, err = dr.Next() for next != "" && err == nil { foundDocIds = append(foundDocIds, next) next, err = dr.Next() } if err != nil { t.Fatal(err) } if !reflect.DeepEqual(expectedDocIds, foundDocIds) { t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds) } err = dr.Close() if err != nil { t.Fatal(err) } //now try again and Advance to skip over "e" dr, err = r.DocIDReader("b", "") if err != nil { t.Fatal(err) } expectedDocIds = []string{"b", "g"} foundDocIds = make([]string, 0) next, err = dr.Next() if err != nil { t.Fatal(err) } else { foundDocIds = append(foundDocIds, next) } next, err = dr.Advance("f") if err != nil { t.Fatal(err) } else { foundDocIds = append(foundDocIds, next) } if !reflect.DeepEqual(expectedDocIds, foundDocIds) { t.Errorf("expected: %v, got %v", expectedDocIds, foundDocIds) } err = dr.Close() if err != nil { t.Fatal(err) } err = r.Close() if err != nil { t.Fatal(err) } }
func TestTermSearcher(t *testing.T) { var queryTerm = "beer" var queryField = "desc" var queryBoost = 3.0 var queryExplain = true analysisQueue := index.NewAnalysisQueue(1) i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } err = i.Open() if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "a", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "b", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "c", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "d", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "e", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "f", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "g", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "h", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "i", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "j", Fields: []document.Field{ document.NewTextField("title", []uint64{}, []byte("cat")), }, }) if err != nil { t.Fatal(err) } indexReader, err := i.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain) if err != nil { t.Fatal(err) } defer func() { err := searcher.Close() if err != nil { t.Fatal(err) } }() searcher.SetQueryNorm(2.0) docCount, err := i.DocCount() if err != nil { t.Fatal(err) } idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0)) expectedQueryWeight := 3 * idf * 3 * idf if expectedQueryWeight != searcher.Weight() { t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight()) } if searcher.Count() != 9 { t.Errorf("expected count of 9, got %d", searcher.Count()) } docMatch, err := searcher.Next() if err != nil { t.Errorf("expected result, got %v", err) } if docMatch.ID != "a" { t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) } docMatch, err = searcher.Advance("c") if err != nil { t.Errorf("expected result, got %v", err) } if docMatch.ID != "c" { t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) } // try advancing past end docMatch, err = searcher.Advance("z") if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } // try pushing next past end docMatch, err = searcher.Next() if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } }
func TestIndexInsertWithStore(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count, and 1 for the back index entry) expectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } if len(storedDoc.Fields) != 1 { t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "test" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } }
func TestLookups(t *testing.T) { aq := index.NewAnalysisQueue(1) f, err := NewFirestorm(gtreap.Name, nil, aq) if err != nil { t.Fatal(err) } err = f.Open() if err != nil { t.Fatal(err) } kvwriter, err := f.(*Firestorm).store.Writer() if err != nil { t.Fatal(err) } rows := []struct { row index.IndexRow garbage bool }{ // needed for warmup to work {NewFieldRow(0, IDFieldName), false}, // 3 documents, with 2 older versions {NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil), true}, {NewTermFreqRow(0, nil, []byte("a"), 2, 0, 0.0, nil), false}, {NewTermFreqRow(0, nil, []byte("b"), 3, 0, 0.0, nil), false}, {NewTermFreqRow(0, nil, []byte("c"), 4, 0, 0.0, nil), true}, {NewTermFreqRow(0, nil, []byte("c"), 5, 0, 0.0, nil), false}, } for _, row := range rows { wb := kvwriter.NewBatch() wb.Set(row.row.Key(), row.row.Value()) err = kvwriter.ExecuteBatch(wb) if err != nil { t.Fatal(err) } // also see the compensator if tfr, ok := row.row.(*TermFreqRow); ok { f.(*Firestorm).compensator.Mutate(tfr.DocID(), tfr.DocNum()) // expect this mutation to be in the in-flight list val := f.(*Firestorm).compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()}) if val == nil { t.Errorf("expected key: % x to be in the inflight list", tfr.DocID()) } f.(*Firestorm).lookuper.lookup(&InFlightItem{docID: tfr.DocID(), docNum: tfr.DocNum()}) // now expect this mutation to NOT be in the in-flight list val = f.(*Firestorm).compensator.inFlight.Get(&InFlightItem{docID: tfr.DocID()}) if val != nil { t.Errorf("expected key: % x to NOT be in the inflight list, got %v", tfr.DocID(), val) } } } // check that doc count is 3 at the end of this docCount, err := f.DocCount() if err != nil { t.Fatal(err) } if docCount != 3 { t.Errorf("expected doc count 3, got %d", docCount) } }
func TestIndexInternalCRUD(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() indexReader, err := idx.Reader() if err != nil { t.Error(err) } // get something that doesn't exist yet val, err := indexReader.GetInternal([]byte("key")) if err != nil { t.Error(err) } if val != nil { t.Errorf("expected nil, got %s", val) } err = indexReader.Close() if err != nil { t.Fatal(err) } // set err = idx.SetInternal([]byte("key"), []byte("abc")) if err != nil { t.Error(err) } indexReader2, err := idx.Reader() if err != nil { t.Error(err) } // get val, err = indexReader2.GetInternal([]byte("key")) if err != nil { t.Error(err) } if string(val) != "abc" { t.Errorf("expected %s, got '%s'", "abc", val) } err = indexReader2.Close() if err != nil { t.Fatal(err) } // delete err = idx.DeleteInternal([]byte("key")) if err != nil { t.Error(err) } indexReader3, err := idx.Reader() if err != nil { t.Error(err) } // get again val, err = indexReader3.GetInternal([]byte("key")) if err != nil { t.Error(err) } if val != nil { t.Errorf("expected nil, got %s", val) } err = indexReader3.Close() if err != nil { t.Fatal(err) } }
func TestWarmupNoGarbage(t *testing.T) { aq := index.NewAnalysisQueue(1) f, err := NewFirestorm(gtreap.Name, nil, aq) if err != nil { t.Fatal(err) } err = f.Open() if err != nil { t.Fatal(err) } kvwriter, err := f.(*Firestorm).store.Writer() if err != nil { t.Fatal(err) } rows := []index.IndexRow{ NewFieldRow(0, IDFieldName), NewTermFreqRow(0, nil, []byte("a"), 1, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("b"), 2, 0, 0.0, nil), NewTermFreqRow(0, nil, []byte("c"), 3, 0, 0.0, nil), } expectedCount := uint64(3) expectedGarbage := uint64(0) for _, row := range rows { wb := kvwriter.NewBatch() wb.Set(row.Key(), row.Value()) err = kvwriter.ExecuteBatch(wb) if err != nil { t.Fatal(err) } } err = kvwriter.Close() if err != nil { t.Fatal(err) } kvreader, err := f.(*Firestorm).store.Reader() if err != nil { t.Fatal(err) } err = f.(*Firestorm).warmup(kvreader) if err != nil { t.Fatal(err) } err = kvreader.Close() if err != nil { t.Fatal(err) } // assert that doc count is correct count, err := f.DocCount() if err != nil { t.Fatal(err) } if count != expectedCount { t.Errorf("expected doc count %d, got %d", expectedCount, count) } // assert that deleted doc numbers size is 0 if f.(*Firestorm).compensator.GarbageCount() != expectedGarbage { t.Errorf("expected 0 deleted doc numbers, got %d", f.(*Firestorm).compensator.GarbageCount()) } // assert that highDocNumber is 3 if f.(*Firestorm).highDocNumber != 3 { t.Errorf("expected highDocNumber to be 3, got %d", f.(*Firestorm).highDocNumber) } }
func TestIndexBatch(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 // first create 2 docs the old fashioned way doc := document.NewDocument("1") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ doc = document.NewDocument("2") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2"))) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ // now create a batch which does 3 things // insert new doc // update existing doc // delete existing doc // net document count change 0 batch := index.NewBatch() doc = document.NewDocument("3") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) batch.Update(doc) doc = document.NewDocument("2") doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated"))) batch.Update(doc) batch.Delete("1") err = idx.Batch(batch) if err != nil { t.Error(err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() docCount := indexReader.DocCount() if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } docIDReader, err := indexReader.DocIDReader("", "") if err != nil { t.Error(err) } docIds := make([]string, 0) docID, err := docIDReader.Next() for docID != "" && err == nil { docIds = append(docIds, docID) docID, err = docIDReader.Next() } if err != nil { t.Error(err) } expectedDocIds := []string{"2", "3"} if !reflect.DeepEqual(docIds, expectedDocIds) { t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds) } }