func TestEnablingDisablingStoringDynamicFields(t *testing.T) {
data := map[string]interface{}{
"name": "bleve",
}
doc := document.NewDocument("x")
mapping := NewIndexMapping()
err := mapping.mapDocument(doc, data)
if err != nil {
t.Fatal(err)
}
for _, field := range doc.Fields {
if field.Name() == "name" && !field.Options().IsStored() {
t.Errorf("expected field 'name' to be stored, isn't")
}
}
StoreDynamic = false
defer func() {
StoreDynamic = true
}()
doc = document.NewDocument("y")
err = mapping.mapDocument(doc, data)
if err != nil {
t.Fatal(err)
}
for _, field := range doc.Fields {
if field.Name() == "name" && field.Options().IsStored() {
t.Errorf("expected field 'name' to be not stored, is")
}
}
}
func TestIndexInsertMultiple(t *testing.T) {
defer os.RemoveAll("test")
store, err := boltdb.Open("test", "bleve")
analysisQueue := NewAnalysisQueue(1)
idx := NewUpsideDownCouch(store, analysisQueue)
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
var expectedCount uint64
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
// should have 4 rows (1 for version, 1 for schema field, and 2 for single term, and 1 for the term count, and 2 for the back index entries)
expectedLength := uint64(1 + 1 + 2 + 1 + 2)
rowCount := idx.rowCount()
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
// close and reopen and and one more to testing counting works correctly
idx.Close()
store, err = boltdb.Open("test", "bleve")
idx = NewUpsideDownCouch(store, analysisQueue)
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer idx.Close()
doc = document.NewDocument("3")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount := idx.DocCount()
if docCount != expectedCount {
t.Errorf("expected doc count: %d, got %d", expectedCount, docCount)
}
}
func TestEnablingDisablingStoringDynamicFields(t *testing.T) {
// first verify that with system defaults, dynamic field is stored
data := map[string]interface{}{
"name": "bleve",
}
doc := document.NewDocument("x")
mapping := NewIndexMapping()
err := mapping.mapDocument(doc, data)
if err != nil {
t.Fatal(err)
}
for _, field := range doc.Fields {
if field.Name() == "name" && !field.Options().IsStored() {
t.Errorf("expected field 'name' to be stored, isn't")
}
}
// now change system level defaults, verify dynamic field is not stored
StoreDynamic = false
defer func() {
StoreDynamic = true
}()
mapping = NewIndexMapping()
doc = document.NewDocument("y")
err = mapping.mapDocument(doc, data)
if err != nil {
t.Fatal(err)
}
for _, field := range doc.Fields {
if field.Name() == "name" && field.Options().IsStored() {
t.Errorf("expected field 'name' to be not stored, is")
}
}
// now override the system level defaults inside the index mapping
mapping = NewIndexMapping()
mapping.StoreDynamic = true
doc = document.NewDocument("y")
err = mapping.mapDocument(doc, data)
if err != nil {
t.Fatal(err)
}
for _, field := range doc.Fields {
if field.Name() == "name" && !field.Options().IsStored() {
t.Errorf("expected field 'name' to be stored, isn't")
}
}
}
func TestIndexInsertThenUpdate(t *testing.T) {
defer os.RemoveAll("test")
store, err := boltdb.Open("test", "bleve")
analysisQueue := NewAnalysisQueue(1)
idx := NewUpsideDownCouch(store, analysisQueue)
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
// this update should overwrite one term, and introduce one new one
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error deleting entry from index: %v", err)
}
// should have 2 row (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry)
expectedLength := uint64(1 + 1 + 2 + 2 + 1)
rowCount := idx.rowCount()
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
// now do another update that should remove one of term
doc = document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error deleting entry from index: %v", err)
}
// should have 2 row (1 for version, 1 for schema field, and 1 for the remaining term, and 1 for the term count, and 1 for the back index entry)
expectedLength = uint64(1 + 1 + 1 + 1 + 1)
rowCount = idx.rowCount()
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
}
func TestIndexDocumentFieldTerms(t *testing.T) {
defer os.RemoveAll("test")
store, err := boltdb.Open("test", "bleve")
if err != nil {
t.Error(err)
}
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
fieldTerms, err := idx.DocumentFieldTerms("1")
if err != nil {
t.Error(err)
}
expectedFieldTerms := index.FieldTerms{
"name": []string{"test"},
"title": []string{"mister"},
}
if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) {
t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms)
}
}
func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewUpsideDownCouch(s, analysisQueue)
err = idx.Open()
if err != nil {
b.Fatal(err)
}
b.StartTimer()
batch := index.NewBatch()
for j := 0; j < 1000; j++ {
if j%batchSize == 0 {
if len(batch.IndexOps) > 0 {
err := idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
batch = index.NewBatch()
}
indexDocument := document.NewDocument("").
AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j)
batch.Update(indexDocument)
}
// close last batch
if len(batch.IndexOps) > 0 {
err := idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
b.StopTimer()
err = idx.Close()
if err != nil {
b.Fatal(err)
}
err = destroy()
if err != nil {
b.Fatal(err)
}
analysisQueue.Close()
}
}
func TestIndexInsertWithStore(t *testing.T) {
defer os.RemoveAll("test")
store, err := boltdb.Open("test", "bleve")
if err != nil {
t.Error(err)
}
analysisQueue := NewAnalysisQueue(1)
idx := NewUpsideDownCouch(store, analysisQueue)
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer idx.Close()
var expectedCount uint64
docCount := idx.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount = idx.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count, and 1 for the back index entry)
expectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1)
rowCount := idx.rowCount()
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
indexReader := idx.Reader()
defer indexReader.Close()
storedDoc, err := indexReader.Document("1")
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 1 {
t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields))
}
textField, ok := storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "test" {
t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
}
}
func (r *firestormReader) Document(id string) (*document.Document, error) {
docID := []byte(id)
docNum, err := r.currDocNumForId(docID)
if err != nil {
return nil, err
} else if docNum == 0 {
return nil, nil
}
rv := document.NewDocument(id)
prefix := StoredPrefixDocIDNum(docID, docNum)
err = visitPrefix(r.r, prefix, func(key, val []byte) (bool, error) {
safeVal := make([]byte, len(val))
copy(safeVal, val)
row, err := NewStoredRowKV(key, safeVal)
if err != nil {
return false, err
}
if row != nil {
fieldName := r.f.fieldCache.FieldIndexed(row.field)
field := r.decodeFieldType(fieldName, row.arrayPositions, row.value.GetRaw())
if field != nil {
rv.AddField(field)
}
}
return true, nil
})
if err != nil {
return nil, err
}
return rv, nil
}
func TestIndexTermReaderCompositeFields(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all")
if err != nil {
t.Error(err)
}
tfd, err := termFieldReader.Next()
for tfd != nil && err == nil {
if tfd.ID != "1" {
t.Errorf("expected to find document id 1")
}
tfd, err = termFieldReader.Next()
}
if err != nil {
t.Error(err)
}
}
func TestMappingJSONWithNull(t *testing.T) {
mapping := NewIndexMapping()
jsonbytes := []byte(`{"name":"marty", "age": null}`)
var jsondoc interface{}
err := json.Unmarshal(jsonbytes, &jsondoc)
if err != nil {
t.Fatal(err)
}
doc := document.NewDocument("1")
err = mapping.mapDocument(doc, jsondoc)
if err != nil {
t.Fatal(err)
}
found := false
count := 0
for _, f := range doc.Fields {
if f.Name() == "name" {
found = true
}
count++
}
if !found {
t.Errorf("expected to find field named 'name'")
}
if count != 1 {
t.Errorf("expected to find 1 find, found %d", count)
}
}
func BenchmarkAnalyze(b *testing.B) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
d := document.NewDocument("1")
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
d.AddField(f)
b.ResetTimer()
for i := 0; i < b.N; i++ {
rv := idx.Analyze(d)
if len(rv.Rows) < 92 || len(rv.Rows) > 93 {
b.Fatalf("expected 512-13 rows, got %d", len(rv.Rows))
}
}
}
func TestMappingPrimitives(t *testing.T) {
tests := []struct {
data interface{}
}{
{data: "marty"},
{data: int(1)},
{data: int8(2)},
{data: int16(3)},
{data: int32(4)},
{data: int64(5)},
{data: uint(6)},
{data: uint8(7)},
{data: uint16(8)},
{data: uint32(9)},
{data: uint64(10)},
{data: float32(11.0)},
{data: float64(12.0)},
{data: false},
}
m := NewIndexMapping()
for _, test := range tests {
doc := document.NewDocument("x")
err := m.MapDocument(doc, test.data)
if err != nil {
t.Fatal(err)
}
if len(doc.Fields) != 1 {
t.Errorf("expected 1 field, got %d for %v", len(doc.Fields), test.data)
}
}
}
func TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) {
type InterfaceThing interface{}
type Thing struct {
InterfaceThing `json:""`
}
x := Thing{
InterfaceThing: map[string]interface{}{
"key": "value",
},
}
doc := document.NewDocument("1")
m := NewIndexMapping()
err := m.MapDocument(doc, x)
if err != nil {
t.Fatal(err)
}
if len(doc.Fields) != 1 {
t.Fatalf("expected 1 field, got %d", len(doc.Fields))
}
if doc.Fields[0].Name() != "key" {
t.Errorf("expected field named 'key', got '%s'", doc.Fields[0].Name())
}
}
func BenchmarkBatch(b *testing.B) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
if err != nil {
b.Fatal(err)
}
err = idx.Open()
if err != nil {
b.Fatal(err)
}
batch := index.NewBatch()
for i := 0; i < 100; i++ {
d := document.NewDocument(strconv.Itoa(i))
f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
d.AddField(f)
batch.Update(d)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
err = idx.Batch(batch)
if err != nil {
b.Fatal(err)
}
}
}
func TestMappingStructWithPointerToString(t *testing.T) {
mapping := buildMapping()
name := "marty"
x := struct {
Name *string
}{
Name: &name,
}
doc := document.NewDocument("1")
err := mapping.mapDocument(doc, x)
if err != nil {
t.Fatal(err)
}
found := false
count := 0
for _, f := range doc.Fields {
if f.Name() == "Name" {
found = true
}
count++
}
if !found {
t.Errorf("expected to find field named 'Name'")
}
if count != 1 {
t.Errorf("expected to find 1 find, found %d", count)
}
}
func TestIndexInsertFields(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(dateField)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
fields, err := indexReader.Fields()
if err != nil {
t.Error(err)
} else {
expectedFields := []string{"name", "age", "unixEpoch"}
if !reflect.DeepEqual(fields, expectedFields) {
t.Errorf("expected fields: %v, got %v", expectedFields, fields)
}
}
}
func TestIndexInsert(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should have 4 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the term count, and 1 for the back index entry)
expectedLength := uint64(1 + 1 + 1 + 1 + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
}
// Index adds the specified index operation to the
// batch. NOTE: the bleve Index is not updated
// until the batch is executed.
func (b *Batch) Index(id string, data interface{}) error {
doc := document.NewDocument(id)
err := b.index.Mapping().mapDocument(doc, data)
if err != nil {
return err
}
b.internal.Update(doc)
return nil
}
func TestIndexDocumentFieldTerms(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewSmolderingCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
fieldTerms, err := indexReader.DocumentFieldTerms(EncodeUvarintAscending(nil, 1), []string{"_id", "name", "title"})
if err != nil {
t.Error(err)
}
expectedFieldTerms := index.FieldTerms{
"name": []string{"test"},
"title": []string{"mister"},
"_id": []string{"1"},
}
if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) {
t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms)
}
}
func TestConcurrentUpdate(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
// do some concurrent updates
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func(i int) {
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), document.StoreField))
err := idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
wg.Done()
}(i)
}
wg.Wait()
// now load the name field and see what we get
r, err := idx.Reader()
if err != nil {
log.Fatal(err)
}
doc, err := r.Document("1")
if err != nil {
log.Fatal(err)
}
if len(doc.Fields) > 1 {
t.Errorf("expected single field, found %d", len(doc.Fields))
}
}
func TestIndexInsert(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
if err != nil {
t.Fatal(err)
}
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
}
func TestIndexFieldsMisc(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewSmolderingCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
fieldName0 := idx.(*SmolderingCouch).fieldCache.FieldIndexed(0)
if fieldName0 != "_id" {
t.Errorf("expected field named '_id', got '%s'", fieldName0)
}
fieldName1 := idx.(*SmolderingCouch).fieldCache.FieldIndexed(1)
if fieldName1 != "name" {
t.Errorf("expected field named 'name', got '%s'", fieldName1)
}
fieldName2 := idx.(*SmolderingCouch).fieldCache.FieldIndexed(2)
if fieldName2 != "title" {
t.Errorf("expected field named 'title', got '%s'", fieldName2)
}
fieldName3 := idx.(*SmolderingCouch).fieldCache.FieldIndexed(3)
if fieldName3 != "" {
t.Errorf("expected field named '', got '%s'", fieldName3)
}
}
func (i *IndexReader) Document(id string) (doc *document.Document, err error) {
// first hit the back index to confirm doc exists
var backIndexRow *BackIndexRow
backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, id)
if err != nil {
return
}
if backIndexRow == nil {
return
}
doc = document.NewDocument(id)
storedRow := NewStoredRow(id, 0, []uint64{}, 'x', nil)
storedRowScanPrefix := storedRow.ScanPrefixForDoc()
it := i.kvreader.Iterator(storedRowScanPrefix)
defer func() {
if cerr := it.Close(); err == nil && cerr != nil {
err = cerr
}
}()
key, val, valid := it.Current()
for valid {
if !bytes.HasPrefix(key, storedRowScanPrefix) {
break
}
safeVal := val
if !i.kvreader.BytesSafeAfterClose() {
safeVal = make([]byte, len(val))
copy(safeVal, val)
}
var row *StoredRow
row, err = NewStoredRowKV(key, safeVal)
if err != nil {
doc = nil
return
}
if row != nil {
fieldName := i.index.fieldIndexCache.FieldName(row.field)
field := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value)
if field != nil {
doc.AddField(field)
}
}
it.Next()
key, val, valid = it.Current()
}
return
}
func TestMappingBug353(t *testing.T) {
dataBytes := `{
"Reviews": [
{
"ReviewID": "RX16692001",
"Content": "Usually stay near the airport..."
}
],
"Other": {
"Inside": "text"
},
"Name": "The Inn at Baltimore White Marsh"
}`
var data map[string]interface{}
err := json.Unmarshal([]byte(dataBytes), &data)
if err != nil {
t.Fatal(err)
}
reviewContentFieldMapping := NewTextFieldMapping()
reviewContentFieldMapping.Analyzer = "crazy"
reviewsMapping := NewDocumentMapping()
reviewsMapping.Dynamic = false
reviewsMapping.AddFieldMappingsAt("Content", reviewContentFieldMapping)
otherMapping := NewDocumentMapping()
otherMapping.Dynamic = false
mapping := NewIndexMapping()
mapping.DefaultMapping.AddSubDocumentMapping("Reviews", reviewsMapping)
mapping.DefaultMapping.AddSubDocumentMapping("Other", otherMapping)
doc := document.NewDocument("x")
err = mapping.mapDocument(doc, data)
if err != nil {
t.Fatal(err)
}
// expect doc has only 2 fields
if len(doc.Fields) != 2 {
t.Errorf("expected doc with 2 fields, got: %d", len(doc.Fields))
for _, f := range doc.Fields {
t.Logf("field named: %s", f.Name())
}
}
}
func CommonBenchmarkIndex(b *testing.B, s store.KVStore) {
index := NewUpsideDownCouch(s)
indexDocument := document.NewDocument("").
AddField(document.NewTextField("body", []uint64{}, []byte("A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.")))
b.ResetTimer()
for i := 0; i < b.N; i++ {
indexDocument.ID = strconv.Itoa(i)
err := index.Update(indexDocument)
if err != nil {
b.Fatal(err)
}
}
}
func TestIndexFieldsMisc(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
store := boltdb.New("test", "bleve")
store.SetMergeOperator(&mergeOperator)
analysisQueue := NewAnalysisQueue(1)
idx := NewUpsideDownCouch(store, analysisQueue)
err := idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
fieldName1 := idx.fieldIndexCache.FieldName(1)
if fieldName1 != "name" {
t.Errorf("expected field named 'name', got '%s'", fieldName1)
}
fieldName2 := idx.fieldIndexCache.FieldName(2)
if fieldName2 != "title" {
t.Errorf("expected field named 'title', got '%s'", fieldName2)
}
fieldName3 := idx.fieldIndexCache.FieldName(3)
if fieldName3 != "" {
t.Errorf("expected field named '', got '%s'", fieldName3)
}
}
func TestDisableDefaultMapping(t *testing.T) {
indexMapping := NewIndexMapping()
indexMapping.DefaultMapping.Enabled = false
data := map[string]string{
"name": "bleve",
}
doc := document.NewDocument("x")
err := indexMapping.mapDocument(doc, data)
if err != nil {
t.Error(err)
}
if len(doc.Fields) > 0 {
t.Errorf("expected no fields, got %d", len(doc.Fields))
}
}
func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers int) {
cache := registry.NewCache()
analyzer, err := cache.AnalyzerNamed("standard")
if err != nil {
b.Fatal(err)
}
indexDocument := document.NewDocument("").
AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer))
b.ResetTimer()
b.StopTimer()
for i := 0; i < b.N; i++ {
s, err := create()
if err != nil {
b.Fatal(err)
}
analysisQueue := index.NewAnalysisQueue(analysisWorkers)
idx := NewUpsideDownCouch(s, analysisQueue)
err = idx.Open()
if err != nil {
b.Fatal(err)
}
indexDocument.ID = strconv.Itoa(i)
// just time the indexing portion
b.StartTimer()
err = idx.Update(indexDocument)
if err != nil {
b.Fatal(err)
}
b.StopTimer()
err = idx.Close()
if err != nil {
b.Fatal(err)
}
err = destroy()
if err != nil {
b.Fatal(err)
}
analysisQueue.Close()
}
}
// Index the object with the specified identifier.
// The IndexMapping for this index will determine
// how the object is indexed.
func (i *indexImpl) Index(id string, data interface{}) error {
i.mutex.RLock()
defer i.mutex.RUnlock()
if !i.open {
return ErrorIndexClosed
}
doc := document.NewDocument(id)
err := i.m.mapDocument(doc, data)
if err != nil {
return err
}
err = i.i.Update(doc)
if err != nil {
return err
}
return nil
}
func TestMappingBool(t *testing.T) {
boolMapping := NewBooleanFieldMapping()
docMapping := NewDocumentMapping()
docMapping.AddFieldMappingsAt("prop", boolMapping)
mapping := NewIndexMapping()
mapping.AddDocumentMapping("doc", docMapping)
pprop := false
x := struct {
Prop bool `json:"prop"`
PProp *bool `json:"pprop"`
}{
Prop: true,
PProp: &pprop,
}
doc := document.NewDocument("1")
err := mapping.mapDocument(doc, x)
if err != nil {
t.Fatal(err)
}
foundProp := false
foundPProp := false
count := 0
for _, f := range doc.Fields {
if f.Name() == "prop" {
foundProp = true
}
if f.Name() == "pprop" {
foundPProp = true
}
count++
}
if !foundProp {
t.Errorf("expected to find bool field named 'prop'")
}
if !foundPProp {
t.Errorf("expected to find pointer to bool field named 'pprop'")
}
if count != 2 {
t.Errorf("expected to find 2 fields, found %d", count)
}
}