func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error { // see if the top level object is a byte array, and possibly run through a converter byteArrayData, ok := data.([]byte) if ok { byteArrayConverterConstructor := registry.ByteArrayConverterByName(im.ByteArrayConverter) if byteArrayConverterConstructor != nil { byteArrayConverter, err := byteArrayConverterConstructor(nil, nil) if err == nil { convertedData, err := byteArrayConverter.Convert(byteArrayData) if err != nil { return err } data = convertedData } else { logger.Printf("error creating byte array converter: %v", err) } } else { logger.Printf("no byte array converter named: %s", im.ByteArrayConverter) } } docType := im.determineType(data) docMapping := im.mappingForType(docType) walkContext := im.newWalkContext(doc, docMapping) docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) // see if the _all field was disabled allMapping := docMapping.documentMappingForPath("_all") if allMapping == nil || (allMapping.Enabled != false) { field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) doc.AddField(field) } return nil }
func TestIndexTermReaderCompositeFields(t *testing.T) { defer func() { err := os.RemoveAll("test") if err != nil { t.Fatal(err) } }() store := boltdb.New("test", "bleve") store.SetMergeOperator(&mergeOperator) analysisQueue := NewAnalysisQueue(1) idx := NewUpsideDownCouch(store, analysisQueue) err := idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") if err != nil { t.Error(err) } tfd, err := termFieldReader.Next() for tfd != nil && err == nil { if tfd.ID != "1" { t.Errorf("expected to find document id 1") } tfd, err = termFieldReader.Next() } if err != nil { t.Error(err) } }
func TestIndexUpdateComposites(t *testing.T) { defer func() { err := os.RemoveAll("test") if err != nil { t.Fatal(err) } }() store := boltdb.New("test", "bleve") store.SetMergeOperator(&mergeOperator) analysisQueue := NewAnalysisQueue(1) idx := NewUpsideDownCouch(store, analysisQueue) err := idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } // should have 72 rows // 1 for version // 3 for schema fields // 4 for text term // 2 for the stored field // 4 for the text term count // 1 for the back index entry expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1) rowCount, err := idx.rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } // now lets update it doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField)) doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() // make sure new values are in index storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testupdated" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } // should have the same row count as before, plus 4 term dictionary garbage rows expectedLength += 4 rowCount, err = idx.rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } }