func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewUpsideDownCouch(s, analysisQueue)
err = idx.Open()
if err != nil {
b.Fatal(err)
}
b.StartTimer()
batch := index.NewBatch()
for j := 0; j < 1000; j++ {
if j%batchSize == 0 {
if len(batch.IndexOps) > 0 {
err := idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
batch = index.NewBatch()
}
indexDocument := document.NewDocument("").
AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j)
batch.Update(indexDocument)
}
// close last batch
if len(batch.IndexOps) > 0 {
err := idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
b.StopTimer()
err = idx.Close()
if err != nil {
b.Fatal(err)
}
err = destroy()
if err != nil {
b.Fatal(err)
}
analysisQueue.Close()
}
}
func BenchmarkBatch(b *testing.B) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
err = idx.Open()
if err != nil {
b.Fatal(err)
}
batch := index.NewBatch()
for i := 0; i < 100; i++ {
d := document.NewDocument(strconv.Itoa(i))
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
d.AddField(f)
batch.Update(d)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
err = idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
}
func TestIndexMetadataRaceBug198(t *testing.T) {
defer func() {
err := os.RemoveAll("testidx")
if err != nil {
t.Fatal(err)
}
}()
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
defer func() {
err := index.Close()
if err != nil {
t.Fatal(err)
}
}()
done := make(chan struct{})
go func() {
for {
select {
case <-done:
return
default:
_, err := index.DocCount()
if err != nil {
t.Fatal(err)
}
}
}
}()
for i := 0; i < 100; i++ {
batch := index.NewBatch()
err = batch.Index("a", []byte("{}"))
if err != nil {
t.Fatal(err)
}
err = index.Batch(batch)
if err != nil {
t.Fatal(err)
}
}
close(done)
}
func TestBatchString(t *testing.T) {
defer func() {
err := os.RemoveAll("testidx")
if err != nil {
t.Fatal(err)
}
}()
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
defer func() {
err := index.Close()
if err != nil {
t.Fatal(err)
}
}()
batch := index.NewBatch()
err = batch.Index("a", []byte("{}"))
if err != nil {
t.Fatal(err)
}
batch.Delete("b")
batch.SetInternal([]byte("c"), []byte{})
batch.DeleteInternal([]byte("d"))
batchStr := batch.String()
if !strings.HasPrefix(batchStr, "Batch (2 ops, 2 internal ops)") {
t.Errorf("expected to start with Batch (2 ops, 2 internal ops), did not")
}
if !strings.Contains(batchStr, "INDEX - 'a'") {
t.Errorf("expected to contain INDEX - 'a', did not")
}
if !strings.Contains(batchStr, "DELETE - 'b'") {
t.Errorf("expected to contain DELETE - 'b', did not")
}
if !strings.Contains(batchStr, "SET INTERNAL - 'c'") {
t.Errorf("expected to contain SET INTERNAL - 'c', did not")
}
if !strings.Contains(batchStr, "DELETE INTERNAL - 'd'") {
t.Errorf("expected to contain DELETE INTERNAL - 'd', did not")
}
}
func TestClosedIndex(t *testing.T) {
index, err := New("", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
err = index.Close()
if err != nil {
t.Fatal(err)
}
err = index.Index("test", "test")
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
err = index.Delete("test")
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
b := index.NewBatch()
err = index.Batch(b)
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
_, err = index.Document("test")
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
_, err = index.DocCount()
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
_, err = index.Search(NewSearchRequest(NewTermQuery("test")))
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
_, err = index.Fields()
if err != ErrorIndexClosed {
t.Errorf("expected error index closed, got %v", err)
}
}
func TestIndexEmptyDocId(t *testing.T) {
defer func() {
err := os.RemoveAll("testidx")
if err != nil {
t.Fatal(err)
}
}()
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
defer func() {
err := index.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := map[string]interface{}{
"body": "nodocid",
}
err = index.Index("", doc)
if err != ErrorEmptyID {
t.Errorf("expect index empty doc id to fail")
}
err = index.Delete("")
if err != ErrorEmptyID {
t.Errorf("expect delete empty doc id to fail")
}
batch := index.NewBatch()
err = batch.Index("", doc)
if err != ErrorEmptyID {
t.Errorf("expect index empty doc id in batch to fail")
}
batch.Delete("")
if batch.Size() > 0 {
t.Errorf("expect delete empty doc id in batch to be ignored")
}
}
func TestBatchReset(t *testing.T) {
defer func() {
err := os.RemoveAll("testidx")
if err != nil {
t.Fatal(err)
}
}()
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
batch := index.NewBatch()
err = batch.Index("k1", struct {
Body string
}{
Body: "v1",
})
if err != nil {
t.Error(err)
}
batch.Delete("k2")
batch.SetInternal([]byte("k3"), []byte("v3"))
batch.DeleteInternal([]byte("k4"))
if batch.Size() != 4 {
t.Logf("%v", batch)
t.Errorf("expected batch size 4, got %d", batch.Size())
}
batch.Reset()
if batch.Size() != 0 {
t.Errorf("expected batch size 0 after reset, got %d", batch.Size())
}
err = index.Close()
if err != nil {
t.Fatal(err)
}
}
// NewBatch creates a new empty batch.
func (i *indexImpl) NewBatch() *Batch {
return &Batch{
index: i,
internal: index.NewBatch(),
}
}
func TestIndexBatch(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
// first create 2 docs the old fashioned way
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
// now create a batch which does 3 things
// insert new doc
// update existing doc
// delete existing doc
// net document count change 0
batch := index.NewBatch()
doc = document.NewDocument("3")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3")))
batch.Update(doc)
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated")))
batch.Update(doc)
batch.Delete("1")
err = idx.Batch(batch)
if err != nil {
t.Error(err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
docCount := indexReader.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
docIDReader, err := indexReader.DocIDReader("", "")
if err != nil {
t.Error(err)
}
docIds := make([]string, 0)
docID, err := docIDReader.Next()
for docID != "" && err == nil {
docIds = append(docIds, docID)
docID, err = docIDReader.Next()
}
if err != nil {
t.Error(err)
}
expectedDocIds := []string{"2", "3"}
if !reflect.DeepEqual(docIds, expectedDocIds) {
t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds)
}
}
func TestIndexCountMatchSearch(t *testing.T) {
defer func() {
err := os.RemoveAll("testidx")
if err != nil {
t.Fatal(err)
}
}()
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func(i int) {
b := index.NewBatch()
for j := 0; j < 200; j++ {
id := fmt.Sprintf("%d", (i*200)+j)
doc := struct {
Body string
}{
Body: "match",
}
err := b.Index(id, doc)
if err != nil {
t.Fatal(err)
}
}
err := index.Batch(b)
if err != nil {
t.Fatal(err)
}
wg.Done()
}(i)
}
wg.Wait()
// search for something that should match all documents
sr, err := index.Search(NewSearchRequest(NewMatchQuery("match")))
if err != nil {
t.Fatal(err)
}
// get the index document count
dc, err := index.DocCount()
if err != nil {
t.Fatal(err)
}
// make sure test is working correctly, doc count should 2000
if dc != 2000 {
t.Errorf("expected doc count 2000, got %d", dc)
}
// make sure our search found all the documents
if dc != sr.Total {
t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc)
}
err = index.Close()
if err != nil {
t.Fatal(err)
}
}
func TestCrud(t *testing.T) {
defer func() {
err := os.RemoveAll("testidx")
if err != nil {
t.Fatal(err)
}
}()
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
doca := map[string]interface{}{
"name": "marty",
"desc": "gophercon india",
}
err = index.Index("a", doca)
if err != nil {
t.Error(err)
}
docy := map[string]interface{}{
"name": "jasper",
"desc": "clojure",
}
err = index.Index("y", docy)
if err != nil {
t.Error(err)
}
err = index.Delete("y")
if err != nil {
t.Error(err)
}
docx := map[string]interface{}{
"name": "rose",
"desc": "googler",
}
err = index.Index("x", docx)
if err != nil {
t.Error(err)
}
err = index.SetInternal([]byte("status"), []byte("pending"))
if err != nil {
t.Error(err)
}
docb := map[string]interface{}{
"name": "steve",
"desc": "cbft master",
}
batch := index.NewBatch()
err = batch.Index("b", docb)
if err != nil {
t.Error(err)
}
batch.Delete("x")
batch.SetInternal([]byte("batchi"), []byte("batchv"))
batch.DeleteInternal([]byte("status"))
err = index.Batch(batch)
if err != nil {
t.Error(err)
}
val, err := index.GetInternal([]byte("batchi"))
if err != nil {
t.Error(err)
}
if string(val) != "batchv" {
t.Errorf("expected 'batchv', got '%s'", val)
}
val, err = index.GetInternal([]byte("status"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got '%s'", val)
}
err = index.SetInternal([]byte("seqno"), []byte("7"))
if err != nil {
t.Error(err)
}
err = index.SetInternal([]byte("status"), []byte("ready"))
if err != nil {
t.Error(err)
}
err = index.DeleteInternal([]byte("status"))
if err != nil {
t.Error(err)
}
val, err = index.GetInternal([]byte("status"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got '%s'", val)
}
val, err = index.GetInternal([]byte("seqno"))
if err != nil {
t.Error(err)
}
if string(val) != "7" {
t.Errorf("expected '7', got '%s'", val)
}
// close the index, open it again, and try some more things
err = index.Close()
if err != nil {
t.Fatal(err)
}
index, err = Open("testidx")
if err != nil {
t.Fatal(err)
}
defer func() {
err := index.Close()
if err != nil {
t.Fatal(err)
}
}()
count, err := index.DocCount()
if err != nil {
t.Fatal(err)
}
if count != 2 {
t.Errorf("expected doc count 2, got %d", count)
}
doc, err := index.Document("a")
if err != nil {
t.Fatal(err)
}
if doc == nil {
t.Errorf("expected doc not nil, got nil")
}
foundNameField := false
for _, field := range doc.Fields {
if field.Name() == "name" && string(field.Value()) == "marty" {
foundNameField = true
}
}
if !foundNameField {
t.Errorf("expected to find field named 'name' with value 'marty'")
}
fields, err := index.Fields()
if err != nil {
t.Fatal(err)
}
expectedFields := map[string]bool{
"_all": false,
"name": false,
"desc": false,
}
if len(fields) != len(expectedFields) {
t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields))
}
for _, f := range fields {
expectedFields[f] = true
}
for ef, efp := range expectedFields {
if !efp {
t.Errorf("field %s is missing", ef)
}
}
}