func TestIndexInsertFields(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() fields, err := indexReader.Fields() if err != nil { t.Error(err) } else { expectedFields := []string{"name", "age", "unixEpoch"} if !reflect.DeepEqual(fields, expectedFields) { t.Errorf("expected fields: %v, got %v", expectedFields, fields) } } }
func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) { fieldName := getFieldName(pathString, path, fm) if fm.Type == "datetime" { options := fm.Options() field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options) if err == nil { context.doc.AddField(field) } else { logger.Printf("could not build date %v", err) } if !fm.IncludeInAll { context.excludedFromAll = append(context.excludedFromAll, fieldName) } } }
func TestIndexInsertFields(t *testing.T) { defer os.RemoveAll("test") store, err := boltdb.Open("test", "bleve") if err != nil { t.Error(err) } idx := NewUpsideDownCouch(store) err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer idx.Close() doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader := idx.Reader() defer indexReader.Close() fields, err := indexReader.Fields() if err != nil { t.Error(err) } else { expectedFields := []string{"name", "age", "unixEpoch"} if !reflect.DeepEqual(fields, expectedFields) { t.Errorf("expected fields: %v, got %v", expectedFields, fields) } } }
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(df) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should have 72 rows // 1 for version // 3 for schema fields // 1 for text term // 16 for numeric terms // 16 for date terms // 3 for the stored field // 1 for the text term count // 16 for numeric term counts // 16 for date term counts // 1 for the back index entry expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1) rowCount, err := idx.(*UpsideDownCouch).rowCount() if err != nil { t.Error(err) } if rowCount != expectedLength { t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } err = indexReader.Close() if err != nil { t.Error(err) } if len(storedDoc.Fields) != 3 { t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "test" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } numField, ok := storedDoc.Fields[1].(*document.NumericField) if !ok { t.Errorf("expected numeric field") } numFieldNumer, err := numField.Number() if err != nil { t.Error(err) } else { if numFieldNumer != 35.99 { t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer) } } dateField, ok := storedDoc.Fields[2].(*document.DateTimeField) if !ok { t.Errorf("expected date field") } dateFieldDate, err := dateField.DateTime() if err != nil { t.Error(err) } else { if dateFieldDate != time.Unix(0, 0).UTC() { t.Errorf("expected date value unix epoch, got %v", dateFieldDate) } } // now update the document, but omit one of the fields doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader2, err := idx.Reader() if err != nil { t.Error(err) } // expected doc count shouldn't have changed docCount = indexReader2.DocCount() if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should only get 2 fields back now though storedDoc, err = indexReader2.Document("1") if err != nil { t.Error(err) } err = indexReader2.Close() if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) } textField, ok = storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testup" { t.Errorf("expected field content 'testup', got '%s'", string(textField.Value())) } numField, ok = storedDoc.Fields[1].(*document.NumericField) if !ok { t.Errorf("expected numeric field") } numFieldNumer, err = numField.Number() if err != nil { t.Error(err) } else { if numFieldNumer != 36.99 { t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer) } } // now delete the document err = idx.Delete("1") expectedCount-- // expected doc count shouldn't have changed docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } }
func TestDump(t *testing.T) { defer os.RemoveAll("test") store, err := boltdb.Open("test", "bleve") if err != nil { t.Error(err) } analysisQueue := NewAnalysisQueue(1) idx := NewUpsideDownCouch(store, analysisQueue) err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer idx.Close() var expectedCount uint64 docCount := idx.DocCount() if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } doc = document.NewDocument("2") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } fieldsCount := 0 fieldsRows := idx.DumpFields() for _ = range fieldsRows { fieldsCount++ } if fieldsCount != 3 { t.Errorf("expected 3 fields, got %d", fieldsCount) } // 1 text term // 16 numeric terms // 16 date terms // 3 stored fields expectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3) docRowCount := 0 docRows := idx.DumpDoc("1") for _ = range docRows { docRowCount++ } if docRowCount != expectedDocRowCount { t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) } docRowCount = 0 docRows = idx.DumpDoc("2") for _ = range docRows { docRowCount++ } if docRowCount != expectedDocRowCount { t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) } // 1 version // fieldsCount field rows // 2 docs * expectedDocRowCount // 2 back index rows // 2 text term row count (2 different text terms) // 16 numeric term row counts (shared for both docs, same numeric value) // 16 date term row counts (shared for both docs, same date value) expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep)))) allRowCount := 0 allRows := idx.DumpAll() for _ = range allRows { allRowCount++ } if allRowCount != expectedAllRowCount { t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount) } }
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { defer func() { err := os.RemoveAll("test") if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(df) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } expectedCount++ err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration) if err != nil { t.Fatal(err) } docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } indexReader, err := idx.Reader() if err != nil { t.Error(err) } storedDoc, err := indexReader.Document("1") if err != nil { t.Error(err) } err = indexReader.Close() if err != nil { t.Error(err) } if len(storedDoc.Fields) != 3 { t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) } textField, ok := storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "test" { t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) } numField, ok := storedDoc.Fields[1].(*document.NumericField) if !ok { t.Errorf("expected numeric field") } numFieldNumer, err := numField.Number() if err != nil { t.Error(err) } else { if numFieldNumer != 35.99 { t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer) } } dateField, ok := storedDoc.Fields[2].(*document.DateTimeField) if !ok { t.Errorf("expected date field") } dateFieldDate, err := dateField.DateTime() if err != nil { t.Error(err) } else { if dateFieldDate != time.Unix(0, 0).UTC() { t.Errorf("expected date value unix epoch, got %v", dateFieldDate) } } // now update the document, but omit one of the fields doc = document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField)) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } indexReader2, err := idx.Reader() if err != nil { t.Error(err) } // expected doc count shouldn't have changed docCount = indexReader2.DocCount() if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } // should only get 2 fields back now though storedDoc, err = indexReader2.Document("1") if err != nil { t.Error(err) } err = indexReader2.Close() if err != nil { t.Error(err) } if len(storedDoc.Fields) != 2 { t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) } textField, ok = storedDoc.Fields[0].(*document.TextField) if !ok { t.Errorf("expected text field") } if string(textField.Value()) != "testup" { t.Errorf("expected field content 'testup', got '%s'", string(textField.Value())) } numField, ok = storedDoc.Fields[1].(*document.NumericField) if !ok { t.Errorf("expected numeric field") } numFieldNumer, err = numField.Number() if err != nil { t.Error(err) } else { if numFieldNumer != 36.99 { t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer) } } // now delete the document err = idx.Delete("1") expectedCount-- err = idx.(*Firestorm).lookuper.waitTasksDone(lookupWaitDuration) if err != nil { t.Fatal(err) } // expected doc count shouldn't have changed docCount, err = idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } }
func TestDump(t *testing.T) { analysisQueue := index.NewAnalysisQueue(1) idx, err := NewFirestorm(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Fatalf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 docCount, err := idx.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } doc = document.NewDocument("2") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } fieldsCount := 0 fieldsRows := idx.DumpFields() for _ = range fieldsRows { fieldsCount++ } if fieldsCount != 4 { // _id field is automatic t.Errorf("expected 4 fields, got %d", fieldsCount) } // 1 id term // 1 text term // 16 numeric terms // 16 date terms // 3 stored fields expectedDocRowCount := int(1 + 1 + (2 * (64 / document.DefaultPrecisionStep)) + 3) docRowCount := 0 docRows := idx.DumpDoc("1") for _ = range docRows { docRowCount++ } if docRowCount != expectedDocRowCount { t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) } docRowCount = 0 docRows = idx.DumpDoc("2") for _ = range docRows { docRowCount++ } if docRowCount != expectedDocRowCount { t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) } err = idx.(*Firestorm).dictUpdater.waitTasksDone(dictWaitDuration) if err != nil { t.Fatal(err) } // 1 version // fieldsCount field rows // 2 docs * expectedDocRowCount // 2 text term row count (2 different text terms) // 16 numeric term row counts (shared for both docs, same numeric value) // 16 date term row counts (shared for both docs, same date value) // expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + int((2 * (64 / document.DefaultPrecisionStep)))) allRowCount := 0 allRows := idx.DumpAll() for _ = range allRows { allRowCount++ } if allRowCount != expectedAllRowCount { t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount) } }
func TestDump(t *testing.T) { defer func() { err := DestroyTest() if err != nil { t.Fatal(err) } }() analysisQueue := index.NewAnalysisQueue(1) idx, err := NewSmolderingCouch(boltdb.Name, boltTestConfig, analysisQueue) if err != nil { t.Fatal(err) } err = idx.Open() if err != nil { t.Errorf("error opening index: %v", err) } defer func() { err := idx.Close() if err != nil { t.Fatal(err) } }() var expectedCount uint64 reader, err := idx.Reader() if err != nil { t.Fatal(err) } docCount, err := reader.DocCount() if err != nil { t.Error(err) } if docCount != expectedCount { t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) } err = reader.Close() if err != nil { t.Fatal(err) } doc := document.NewDocument("1") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } doc = document.NewDocument("2") doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField)) doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) if err != nil { t.Error(err) } doc.AddField(dateField) err = idx.Update(doc) if err != nil { t.Errorf("Error updating index: %v", err) } fieldsCount := 0 reader, err = idx.Reader() if err != nil { t.Fatal(err) } fieldsRows := reader.DumpFields() for range fieldsRows { fieldsCount++ } if fieldsCount != 4 { t.Errorf("expected 4 fields, got %d", fieldsCount) } // 1 text term // 16 numeric terms // 16 date terms // 3 stored fields // 1 id term row // 1 id stored row expectedDocRowCount := int(1+(2*(64/document.DefaultPrecisionStep))+3) + 1 + 1 docRowCount := 0 docRows := reader.DumpDoc("1") for range docRows { docRowCount++ } if docRowCount != expectedDocRowCount { t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) } docRowCount = 0 docRows = reader.DumpDoc("2") for range docRows { docRowCount++ } if docRowCount != expectedDocRowCount { t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) } // 1 version // fieldsCount field rows // 2 docs * expectedDocRowCount // 2 back index rows //4 text term row count (4 different text terms) // 16 numeric term row counts (shared for both docs, same numeric value) // 16 date term row counts (shared for both docs, same date value) expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 4 + int((2 * (64 / document.DefaultPrecisionStep)))) allRowCount := 0 allRows := reader.DumpAll() for range allRows { allRowCount++ } if allRowCount != expectedAllRowCount { t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount) } err = reader.Close() if err != nil { t.Fatal(err) } }