func newIndex(path string, mapping *IndexMapping) (*indexImpl, error) {
// first validate the mapping
err := mapping.validate()
if err != nil {
return nil, err
}
if path == "" {
return newMemIndex(mapping)
}
rv := indexImpl{
path: path,
m: mapping,
meta: newIndexMeta(Config.DefaultKVStore),
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// at this point there hope we can be successful, so save index meta
err = rv.meta.Save(path)
if err != nil {
return nil, err
}
storeConfig := map[string]interface{}{
"path": indexStorePath(path),
"create_if_missing": true,
"error_if_exists": true,
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func init() {
inMemStore, _ := inmem.Open()
twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore)
for _, doc := range twoDocIndexDocs {
twoDocIndex.Update(doc)
}
}
func init() {
inMemStore, _ := inmem.Open()
analysisQueue := upside_down.NewAnalysisQueue(1)
twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore, analysisQueue)
for _, doc := range twoDocIndexDocs {
twoDocIndex.Update(doc)
}
}
func init() {
inMemStore, _ := inmem.New()
analysisQueue := index.NewAnalysisQueue(1)
twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore, analysisQueue)
err := twoDocIndex.Open()
if err != nil {
panic(err)
}
for _, doc := range twoDocIndexDocs {
err := twoDocIndex.Update(doc)
if err != nil {
panic(err)
}
}
}
func init() {
analysisQueue := index.NewAnalysisQueue(1)
var err error
twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
if err != nil {
panic(err)
}
err = twoDocIndex.Open()
if err != nil {
panic(err)
}
for _, doc := range twoDocIndexDocs {
err := twoDocIndex.Update(doc)
if err != nil {
panic(err)
}
}
}
func newMemIndex(mapping *IndexMapping) (*indexImpl, error) {
rv := indexImpl{
path: "",
m: mapping,
meta: newIndexMeta("mem", nil),
stats: &IndexStat{},
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// now open the store
var err error
rv.s, err = storeConstructor(nil)
if err != nil {
return nil, err
}
// open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue)
err = rv.i.Open()
if err != nil {
return nil, err
}
rv.stats.indexStat = rv.i.Stats()
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func newIndexUsing(path string, mapping *IndexMapping, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
// first validate the mapping
err := mapping.validate()
if err != nil {
return nil, err
}
if path == "" {
return newMemIndex(mapping)
}
if kvconfig == nil {
kvconfig = map[string]interface{}{}
}
rv := indexImpl{
path: path,
m: mapping,
meta: newIndexMeta(kvstore, kvconfig),
stats: &IndexStat{},
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// at this point there is hope that we can be successful, so save index meta
err = rv.meta.Save(path)
if err != nil {
return nil, err
}
kvconfig["create_if_missing"] = true
kvconfig["error_if_exists"] = true
kvconfig["path"] = indexStorePath(path)
// now create the store
rv.s, err = storeConstructor(kvconfig)
if err != nil {
return nil, err
}
// open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue)
err = rv.i.Open()
if err != nil {
return nil, err
}
rv.stats.indexStat = rv.i.Stats()
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) {
rv = &indexImpl{
path: path,
stats: &IndexStat{},
}
rv.meta, err = openIndexMeta(path)
if err != nil {
return nil, err
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
storeConfig := rv.meta.Config
if storeConfig == nil {
storeConfig = map[string]interface{}{}
}
storeConfig["path"] = indexStorePath(path)
storeConfig["create_if_missing"] = false
storeConfig["error_if_exists"] = false
for rck, rcv := range runtimeConfig {
storeConfig[rck] = rcv
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue)
err = rv.i.Open()
if err != nil {
return nil, err
}
rv.stats.indexStat = rv.i.Stats()
// now load the mapping
indexReader, err := rv.i.Reader()
if err != nil {
return nil, err
}
defer func() {
if cerr := indexReader.Close(); cerr != nil && err == nil {
err = cerr
}
}()
mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
if err != nil {
return nil, err
}
var im IndexMapping
err = json.Unmarshal(mappingBytes, &im)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
// validate the mapping
err = im.validate()
if err != nil {
// note even if the mapping is invalid
// we still return an open usable index
return rv, err
}
rv.m = &im
return rv, err
}
func TestTermSearcher(t *testing.T) {
var queryTerm = "beer"
var queryField = "desc"
var queryBoost = 3.0
var queryExplain = true
inMemStore, _ := inmem.Open()
i := upside_down.NewUpsideDownCouch(inMemStore)
i.Update(&document.Document{
ID: "a",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "b",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "c",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "d",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "e",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "f",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "g",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "h",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "i",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
i.Update(&document.Document{
ID: "j",
Fields: []document.Field{
document.NewTextField("title", []uint64{}, []byte("cat")),
},
})
indexReader := i.Reader()
defer indexReader.Close()
searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain)
if err != nil {
t.Fatal(err)
}
defer searcher.Close()
searcher.SetQueryNorm(2.0)
idf := 1.0 + math.Log(float64(i.DocCount())/float64(searcher.Count()+1.0))
expectedQueryWeight := 3 * idf * 3 * idf
if expectedQueryWeight != searcher.Weight() {
t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight())
}
if searcher.Count() != 9 {
t.Errorf("expected count of 9, got %d", searcher.Count())
}
docMatch, err := searcher.Next()
if err != nil {
t.Errorf("expected result, got %v", err)
}
if docMatch.ID != "a" {
t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID)
}
docMatch, err = searcher.Advance("c")
if err != nil {
t.Errorf("expected result, got %v", err)
}
if docMatch.ID != "c" {
t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID)
}
// try advancing past end
docMatch, err = searcher.Advance("z")
if err != nil {
t.Fatal(err)
}
if docMatch != nil {
t.Errorf("expected nil, got %v", docMatch)
}
// try pushing next past end
docMatch, err = searcher.Next()
if err != nil {
t.Fatal(err)
}
if docMatch != nil {
t.Errorf("expected nil, got %v", docMatch)
}
}
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
analysisQueue := index.NewAnalysisQueue(1)
i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = i.Open()
if err != nil {
t.Fatal(err)
}
for _, id := range indexed {
err = i.Update(&document.Document{
ID: id,
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
}
indexReader, err := i.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
searcher, err := NewDocIDSearcher(indexReader, searched, 1.0, false)
if err != nil {
t.Fatal(err)
}
defer func() {
err := searcher.Close()
if err != nil {
t.Fatal(err)
}
}()
if searcher.Count() != uint64(len(wanted)) {
t.Fatalf("expected count %v got %v", len(wanted), searcher.Count())
}
// Check the sequence
for i, id := range wanted {
m, err := searcher.Next()
if err != nil {
t.Fatal(err)
}
if id != m.ID {
t.Fatalf("expected %v at position %v, got %v", id, i, m.ID)
}
}
m, err := searcher.Next()
if err != nil {
t.Fatal(err)
}
if m != nil {
t.Fatalf("expected nil past the end of the sequence, got %v", m.ID)
}
// Check seeking
for _, id := range wanted {
if len(id) != 2 {
t.Fatalf("expected identifier must be 2 characters long, got %v", id)
}
before := id[:1]
for _, target := range []string{before, id} {
m, err := searcher.Advance(target)
if err != nil {
t.Fatal(err)
}
if m == nil || m.ID != id {
t.Fatalf("advancing to %v returned %v instead of %v", before, m, id)
}
}
}
// Seek after the end of the sequence
after := "zzz"
m, err = searcher.Advance(after)
if err != nil {
t.Fatal(err)
}
if m != nil {
t.Fatalf("advancing past the end of the sequence should return nil, got %v", m)
}
}
func openIndex(path string) (*indexImpl, error) {
rv := indexImpl{
path: path,
}
var err error
rv.meta, err = openIndexMeta(path)
if err != nil {
return nil, err
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
storeConfig := map[string]interface{}{
"path": indexStorePath(path),
"create_if_missing": false,
"error_if_exists": false,
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now load the mapping
indexReader := rv.i.Reader()
defer indexReader.Close()
mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
if err != nil {
return nil, err
}
var im IndexMapping
err = json.Unmarshal(mappingBytes, &im)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
// validate the mapping
err = im.validate()
if err != nil {
// note even if the mapping is invalid
// we still return an open usable index
return &rv, err
}
rv.m = &im
return &rv, nil
}
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
analysisQueue := index.NewAnalysisQueue(1)
i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = i.Open()
if err != nil {
t.Fatal(err)
}
for _, id := range indexed {
err = i.Update(&document.Document{
ID: id,
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
}
indexReader, err := i.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
searcher, err := NewDocIDSearcher(indexReader, searched, 1.0, false)
if err != nil {
t.Fatal(err)
}
defer func() {
err := searcher.Close()
if err != nil {
t.Fatal(err)
}
}()
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0),
}
// Check the sequence
for i, id := range wanted {
m, err := searcher.Next(ctx)
if err != nil {
t.Fatal(err)
}
if !index.IndexInternalID(id).Equals(m.IndexInternalID) {
t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID)
}
ctx.DocumentMatchPool.Put(m)
}
m, err := searcher.Next(ctx)
if err != nil {
t.Fatal(err)
}
if m != nil {
t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID)
}
ctx.DocumentMatchPool.Put(m)
// Check seeking
for _, id := range wanted {
if len(id) != 2 {
t.Fatalf("expected identifier must be 2 characters long, got %v", id)
}
before := id[:1]
for _, target := range []string{before, id} {
m, err := searcher.Advance(ctx, index.IndexInternalID(target))
if err != nil {
t.Fatal(err)
}
if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) {
t.Fatalf("advancing to %v returned %v instead of %v", before, m, id)
}
ctx.DocumentMatchPool.Put(m)
}
}
// Seek after the end of the sequence
after := "zzz"
m, err = searcher.Advance(ctx, index.IndexInternalID(after))
if err != nil {
t.Fatal(err)
}
if m != nil {
t.Fatalf("advancing past the end of the sequence should return nil, got %v", m)
}
ctx.DocumentMatchPool.Put(m)
}
func TestTermSearcher(t *testing.T) {
var queryTerm = "beer"
var queryField = "desc"
var queryBoost = 3.0
var queryExplain = true
analysisQueue := index.NewAnalysisQueue(1)
i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
if err != nil {
t.Fatal(err)
}
err = i.Open()
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "a",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "b",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "c",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "d",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "e",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "f",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "g",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "h",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "i",
Fields: []document.Field{
document.NewTextField("desc", []uint64{}, []byte("beer")),
},
})
if err != nil {
t.Fatal(err)
}
err = i.Update(&document.Document{
ID: "j",
Fields: []document.Field{
document.NewTextField("title", []uint64{}, []byte("cat")),
},
})
if err != nil {
t.Fatal(err)
}
indexReader, err := i.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := indexReader.Close()
if err != nil {
t.Fatal(err)
}
}()
searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain)
if err != nil {
t.Fatal(err)
}
defer func() {
err := searcher.Close()
if err != nil {
t.Fatal(err)
}
}()
searcher.SetQueryNorm(2.0)
docCount, err := indexReader.DocCount()
if err != nil {
t.Fatal(err)
}
idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0))
expectedQueryWeight := 3 * idf * 3 * idf
if expectedQueryWeight != searcher.Weight() {
t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight())
}
if searcher.Count() != 9 {
t.Errorf("expected count of 9, got %d", searcher.Count())
}
ctx := &search.SearchContext{
DocumentMatchPool: search.NewDocumentMatchPool(1, 0),
}
docMatch, err := searcher.Next(ctx)
if err != nil {
t.Errorf("expected result, got %v", err)
}
if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) {
t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID)
}
ctx.DocumentMatchPool.Put(docMatch)
docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c"))
if err != nil {
t.Errorf("expected result, got %v", err)
}
if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) {
t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID)
}
// try advancing past end
ctx.DocumentMatchPool.Put(docMatch)
docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z"))
if err != nil {
t.Fatal(err)
}
if docMatch != nil {
t.Errorf("expected nil, got %v", docMatch)
}
// try pushing next past end
ctx.DocumentMatchPool.Put(docMatch)
docMatch, err = searcher.Next(ctx)
if err != nil {
t.Fatal(err)
}
if docMatch != nil {
t.Errorf("expected nil, got %v", docMatch)
}
}