func TestIndexInsertFields(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(dateField)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
fields, err := indexReader.Fields()
if err != nil {
t.Error(err)
} else {
expectedFields := []string{"name", "age", "unixEpoch"}
if !reflect.DeepEqual(fields, expectedFields) {
t.Errorf("expected fields: %v, got %v", expectedFields, fields)
}
}
}
func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) {
fieldName := getFieldName(pathString, path, fm)
if fm.Type == "datetime" {
options := fm.Options()
field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options)
if err == nil {
context.doc.AddField(field)
} else {
logger.Printf("could not build date %v", err)
}
if !fm.IncludeInAll {
context.excludedFromAll = append(context.excludedFromAll, fieldName)
}
}
}
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
defer func() {
err := DestroyTest()
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(df)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should have 72 rows
// 1 for version
// 3 for schema fields
// 1 for text term
// 16 for numeric terms
// 16 for date terms
// 3 for the stored field
// 1 for the text term count
// 16 for numeric term counts
// 16 for date term counts
// 1 for the back index entry
expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1)
rowCount, err := idx.(*UpsideDownCouch).rowCount()
if err != nil {
t.Error(err)
}
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
storedDoc, err := indexReader.Document("1")
if err != nil {
t.Error(err)
}
err = indexReader.Close()
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 3 {
t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields))
}
textField, ok := storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "test" {
t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
}
numField, ok := storedDoc.Fields[1].(*document.NumericField)
if !ok {
t.Errorf("expected numeric field")
}
numFieldNumer, err := numField.Number()
if err != nil {
t.Error(err)
} else {
if numFieldNumer != 35.99 {
t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer)
}
}
dateField, ok := storedDoc.Fields[2].(*document.DateTimeField)
if !ok {
t.Errorf("expected date field")
}
dateFieldDate, err := dateField.DateTime()
if err != nil {
t.Error(err)
} else {
if dateFieldDate != time.Unix(0, 0).UTC() {
t.Errorf("expected date value unix epoch, got %v", dateFieldDate)
}
}
// now update the document, but omit one of the fields
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader2, err := idx.Reader()
if err != nil {
t.Error(err)
}
// expected doc count shouldn't have changed
docCount = indexReader2.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should only get 2 fields back now though
storedDoc, err = indexReader2.Document("1")
if err != nil {
t.Error(err)
}
err = indexReader2.Close()
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 2 {
t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields))
}
textField, ok = storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "testup" {
t.Errorf("expected field content 'testup', got '%s'", string(textField.Value()))
}
numField, ok = storedDoc.Fields[1].(*document.NumericField)
if !ok {
t.Errorf("expected numeric field")
}
numFieldNumer, err = numField.Number()
if err != nil {
t.Error(err)
} else {
if numFieldNumer != 36.99 {
t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer)
}
}
// now delete the document
err = idx.Delete("1")
expectedCount--
// expected doc count shouldn't have changed
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
}
func TestDump(t *testing.T) {
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(gtreap.Name, nil, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Fatalf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(dateField)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
doc = document.NewDocument("2")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(dateField)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
fieldsCount := 0
fieldsRows := idx.DumpFields()
for _ = range fieldsRows {
fieldsCount++
}
if fieldsCount != 4 { // _id field is automatic
t.Errorf("expected 4 fields, got %d", fieldsCount)
}
// 1 id term
// 1 text term
// 16 numeric terms
// 16 date terms
// 3 stored fields
expectedDocRowCount := int(1 + 1 + (2 * (64 / document.DefaultPrecisionStep)) + 3)
docRowCount := 0
docRows := idx.DumpDoc("1")
for _ = range docRows {
docRowCount++
}
if docRowCount != expectedDocRowCount {
t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
}
docRowCount = 0
docRows = idx.DumpDoc("2")
for _ = range docRows {
docRowCount++
}
if docRowCount != expectedDocRowCount {
t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
}
err = idx.(*Firestorm).dictUpdater.waitTasksDone(dictWaitDuration)
if err != nil {
t.Fatal(err)
}
// 1 version
// fieldsCount field rows
// 2 docs * expectedDocRowCount
// 2 text term row count (2 different text terms)
// 16 numeric term row counts (shared for both docs, same numeric value)
// 16 date term row counts (shared for both docs, same date value)
//
expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + int((2 * (64 / document.DefaultPrecisionStep))))
allRowCount := 0
allRows := idx.DumpAll()
for _ = range allRows {
allRowCount++
}
if allRowCount != expectedAllRowCount {
t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount)
}
}
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
analysisQueue := index.NewAnalysisQueue(1)
idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = idx.Open()
if err != nil {
t.Errorf("error opening index: %v", err)
}
defer func() {
err := idx.Close()
if err != nil {
t.Fatal(err)
}
}()
var expectedCount uint64
docCount, err := idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
doc.AddField(df)
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
expectedCount++
err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
if err != nil {
t.Fatal(err)
}
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
indexReader, err := idx.Reader()
if err != nil {
t.Error(err)
}
storedDoc, err := indexReader.Document("1")
if err != nil {
t.Error(err)
}
err = indexReader.Close()
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 3 {
t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields))
}
textField, ok := storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "test" {
t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
}
numField, ok := storedDoc.Fields[1].(*document.NumericField)
if !ok {
t.Errorf("expected numeric field")
}
numFieldNumer, err := numField.Number()
if err != nil {
t.Error(err)
} else {
if numFieldNumer != 35.99 {
t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer)
}
}
dateField, ok := storedDoc.Fields[2].(*document.DateTimeField)
if !ok {
t.Errorf("expected date field")
}
dateFieldDate, err := dateField.DateTime()
if err != nil {
t.Error(err)
} else {
if dateFieldDate != time.Unix(0, 0).UTC() {
t.Errorf("expected date value unix epoch, got %v", dateFieldDate)
}
}
// now update the document, but omit one of the fields
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
}
indexReader2, err := idx.Reader()
if err != nil {
t.Error(err)
}
// expected doc count shouldn't have changed
docCount = indexReader2.DocCount()
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
// should only get 2 fields back now though
storedDoc, err = indexReader2.Document("1")
if err != nil {
t.Error(err)
}
err = indexReader2.Close()
if err != nil {
t.Error(err)
}
if len(storedDoc.Fields) != 2 {
t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields))
}
textField, ok = storedDoc.Fields[0].(*document.TextField)
if !ok {
t.Errorf("expected text field")
}
if string(textField.Value()) != "testup" {
t.Errorf("expected field content 'testup', got '%s'", string(textField.Value()))
}
numField, ok = storedDoc.Fields[1].(*document.NumericField)
if !ok {
t.Errorf("expected numeric field")
}
numFieldNumer, err = numField.Number()
if err != nil {
t.Error(err)
} else {
if numFieldNumer != 36.99 {
t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer)
}
}
// now delete the document
err = idx.Delete("1")
expectedCount--
err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration)
if err != nil {
t.Fatal(err)
}
// expected doc count shouldn't have changed
docCount, err = idx.DocCount()
if err != nil {
t.Error(err)
}
if docCount != expectedCount {
t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
}
}