func TestIndexTermReaderCompositeFields(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") if err != nil { t.Error(err) } tfd, err := termFieldReader.Next() for tfd != nil && err == nil { if tfd.ID != "1" { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next() } if err != nil { t.Error(err) } }
func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error { // see if the top level object is a byte array, and possibly run through conveter byteArrayData, ok := data.([]byte) if ok { byteArrayConverterConstructor := registry.ByteArrayConverterByName(im.ByteArrayConverter) if byteArrayConverterConstructor != nil { byteArrayConverter, err := byteArrayConverterConstructor(nil, nil) if err == nil { convertedData, err := byteArrayConverter.Convert(byteArrayData) if err != nil { return err } data = convertedData } else { log.Printf("error creating byte array converter: %v", err) } } else { log.Printf("no byte array converter named: %s", im.ByteArrayConverter) } } docType := im.determineType(data) docMapping := im.mappingForType(docType) walkContext := im.newWalkContext(doc, docMapping) docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) // see if the _all field was disabled allMapping := docMapping.documentMappingForPath("_all") if allMapping == nil || (allMapping.Enabled != false) { field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) doc.AddField(field) } return nil }
func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error { docType := im.determineType(data) docMapping := im.mappingForType(docType) walkContext := im.newWalkContext(doc, docMapping) if docMapping.Enabled { docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) // see if the _all field was disabled allMapping := docMapping.documentMappingForPath("_all") if allMapping == nil || (allMapping.Enabled != false) { field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) doc.AddField(field) } } return nil }
func TestIndexTermReaderCompositeFields(t *testing.T) { defer os.RemoveAll("test") store, err := boltdb.Open("test", "bleve") if err != nil { t.Error(err) } idx := NewUpsideDownCouch(store) err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer idx.Close() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader := idx.Reader() defer indexReader.Close() termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") if err != nil { t.Error(err) } tfd, err := termFieldReader.Next() for tfd != nil && err == nil { if tfd.ID != "1" { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next() } if err != nil { t.Error(err) } }
func TestAnalysisBug328(t *testing.T) { cache := registry.NewCache() analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name) if err != nil { t.Fatal(err) } analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } d := document.NewDocument("1") f := document.NewTextFieldCustom("title", nil, []byte("bleve"), document.IndexField|document.IncludeTermVectors, analyzer) d.AddField(f) f = document.NewTextFieldCustom("body", nil, []byte("bleve"), document.IndexField|document.IncludeTermVectors, analyzer) d.AddField(f) cf := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, []string{}, document.IndexField|document.IncludeTermVectors) d.AddField(cf) rv := idx.Analyze(d) fieldIndexes := make(map[uint16]string) for _, row := range rv.Rows { if row, ok := row.(*FieldRow); ok { fieldIndexes[row.index] = row.Name() } if row, ok := row.(*TermFreqRow); ok && string(row.term) == "bleve" { for _, vec := range row.Vectors() { if vec.GetField() != uint32(row.field) { if fieldIndexes[row.field] != "_all" { t.Errorf("row named %s field %d - vector field %d", fieldIndexes[row.field], row.field, vec.GetField()) } } } } } }
func TestIndexUpdateComposites(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } // should have 72 rows // 1 for version // 3 for schema fields // 4 for text term // 2 for the stored field // 4 for the text term count // 1 for the back index entry expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } // now lets update it doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() // make sure new values are in index storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testupdated" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } // should have the same row count as before, plus 4 term dictionary garbage rows expectedLength += 4 rowCount, err = idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } }
func TestIndexUpdateComposites(t *testing.T) { defer os.RemoveAll("test") store, err := boltdb.Open("test", "bleve") if err != nil { t.Error(err) } idx := NewUpsideDownCouch(store) err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer idx.Close() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } // should have 72 rows // 1 for version // 3 for schema fields // 4 for text term // 2 for the stored field // 4 for the text term count // 1 for the back index entry expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1) rowCount := idx.rowCount() if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } // now lets update it doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader := idx.Reader() defer indexReader.Close() // make sure new values are in index storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testupdated" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } // should have the same row count as before rowCount = idx.rowCount() if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } }
func TestIndexUpdateComposites(t *testing.T) { defer func() { err := os.RemoveAll("test") if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } // now lets update it doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() // make sure new values are in index storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testupdated" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } }