func newIndex(path string, mapping *IndexMapping) (*indexImpl, error) { // first validate the mapping err := mapping.validate() if err != nil { return nil, err } if path == "" { return newMemIndex(mapping) } rv := indexImpl{ path: path, m: mapping, meta: newIndexMeta(Config.DefaultKVStore), } storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) if storeConstructor == nil { return nil, ErrorUnknownStorageType } // at this point there hope we can be successful, so save index meta err = rv.meta.Save(path) if err != nil { return nil, err } storeConfig := map[string]interface{}{ "path": indexStorePath(path), "create_if_missing": true, "error_if_exists": true, } // now open the store rv.s, err = storeConstructor(storeConfig) if err != nil { return nil, err } // open open the index rv.i = upside_down.NewUpsideDownCouch(rv.s) err = rv.i.Open() if err != nil { return nil, err } // now persist the mapping mappingBytes, err := json.Marshal(mapping) if err != nil { return nil, err } err = rv.i.SetInternal(mappingInternalKey, mappingBytes) if err != nil { return nil, err } // mark the index as open rv.mutex.Lock() defer rv.mutex.Unlock() rv.open = true return &rv, nil }
func init() { inMemStore, _ := inmem.Open() twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore) for _, doc := range twoDocIndexDocs { twoDocIndex.Update(doc) } }
func init() { inMemStore, _ := inmem.Open() analysisQueue := upside_down.NewAnalysisQueue(1) twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore, analysisQueue) for _, doc := range twoDocIndexDocs { twoDocIndex.Update(doc) } }
func init() { inMemStore, _ := inmem.New() analysisQueue := index.NewAnalysisQueue(1) twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore, analysisQueue) err := twoDocIndex.Open() if err != nil { panic(err) } for _, doc := range twoDocIndexDocs { err := twoDocIndex.Update(doc) if err != nil { panic(err) } } }
func init() { analysisQueue := index.NewAnalysisQueue(1) var err error twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { panic(err) } err = twoDocIndex.Open() if err != nil { panic(err) } for _, doc := range twoDocIndexDocs { err := twoDocIndex.Update(doc) if err != nil { panic(err) } } }
func newMemIndex(mapping *IndexMapping) (*indexImpl, error) { rv := indexImpl{ path: "", m: mapping, meta: newIndexMeta("mem", nil), stats: &IndexStat{}, } storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) if storeConstructor == nil { return nil, ErrorUnknownStorageType } // now open the store var err error rv.s, err = storeConstructor(nil) if err != nil { return nil, err } // open the index rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue) err = rv.i.Open() if err != nil { return nil, err } rv.stats.indexStat = rv.i.Stats() // now persist the mapping mappingBytes, err := json.Marshal(mapping) if err != nil { return nil, err } err = rv.i.SetInternal(mappingInternalKey, mappingBytes) if err != nil { return nil, err } // mark the index as open rv.mutex.Lock() defer rv.mutex.Unlock() rv.open = true return &rv, nil }
func newIndexUsing(path string, mapping *IndexMapping, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) { // first validate the mapping err := mapping.validate() if err != nil { return nil, err } if path == "" { return newMemIndex(mapping) } if kvconfig == nil { kvconfig = map[string]interface{}{} } rv := indexImpl{ path: path, m: mapping, meta: newIndexMeta(kvstore, kvconfig), stats: &IndexStat{}, } storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) if storeConstructor == nil { return nil, ErrorUnknownStorageType } // at this point there is hope that we can be successful, so save index meta err = rv.meta.Save(path) if err != nil { return nil, err } kvconfig["create_if_missing"] = true kvconfig["error_if_exists"] = true kvconfig["path"] = indexStorePath(path) // now create the store rv.s, err = storeConstructor(kvconfig) if err != nil { return nil, err } // open the index rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue) err = rv.i.Open() if err != nil { return nil, err } rv.stats.indexStat = rv.i.Stats() // now persist the mapping mappingBytes, err := json.Marshal(mapping) if err != nil { return nil, err } err = rv.i.SetInternal(mappingInternalKey, mappingBytes) if err != nil { return nil, err } // mark the index as open rv.mutex.Lock() defer rv.mutex.Unlock() rv.open = true return &rv, nil }
func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) { rv = &indexImpl{ path: path, stats: &IndexStat{}, } rv.meta, err = openIndexMeta(path) if err != nil { return nil, err } storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) if storeConstructor == nil { return nil, ErrorUnknownStorageType } storeConfig := rv.meta.Config if storeConfig == nil { storeConfig = map[string]interface{}{} } storeConfig["path"] = indexStorePath(path) storeConfig["create_if_missing"] = false storeConfig["error_if_exists"] = false for rck, rcv := range runtimeConfig { storeConfig[rck] = rcv } // now open the store rv.s, err = storeConstructor(storeConfig) if err != nil { return nil, err } // open the index rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue) err = rv.i.Open() if err != nil { return nil, err } rv.stats.indexStat = rv.i.Stats() // now load the mapping indexReader, err := rv.i.Reader() if err != nil { return nil, err } defer func() { if cerr := indexReader.Close(); cerr != nil && err == nil { err = cerr } }() mappingBytes, err := indexReader.GetInternal(mappingInternalKey) if err != nil { return nil, err } var im IndexMapping err = json.Unmarshal(mappingBytes, &im) if err != nil { return nil, err } // mark the index as open rv.mutex.Lock() defer rv.mutex.Unlock() rv.open = true // validate the mapping err = im.validate() if err != nil { // note even if the mapping is invalid // we still return an open usable index return rv, err } rv.m = &im return rv, err }
func TestTermSearcher(t *testing.T) { var queryTerm = "beer" var queryField = "desc" var queryBoost = 3.0 var queryExplain = true inMemStore, _ := inmem.Open() i := upside_down.NewUpsideDownCouch(inMemStore) i.Update(&document.Document{ ID: "a", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "b", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "c", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "d", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "e", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "f", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "g", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "h", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "i", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) i.Update(&document.Document{ ID: "j", Fields: []document.Field{ document.NewTextField("title", []uint64{}, []byte("cat")), }, }) indexReader := i.Reader() defer indexReader.Close() searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain) if err != nil { t.Fatal(err) } defer searcher.Close() searcher.SetQueryNorm(2.0) idf := 1.0 + math.Log(float64(i.DocCount())/float64(searcher.Count()+1.0)) expectedQueryWeight := 3 * idf * 3 * idf if expectedQueryWeight != searcher.Weight() { t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight()) } if searcher.Count() != 9 { t.Errorf("expected count of 9, got %d", searcher.Count()) } docMatch, err := searcher.Next() if err != nil { t.Errorf("expected result, got %v", err) } if docMatch.ID != "a" { t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) } docMatch, err = searcher.Advance("c") if err != nil { t.Errorf("expected result, got %v", err) } if docMatch.ID != "c" { t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) } // try advancing past end docMatch, err = searcher.Advance("z") if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } // try pushing next past end docMatch, err = searcher.Next() if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } }
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { analysisQueue := index.NewAnalysisQueue(1) i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } err = i.Open() if err != nil { t.Fatal(err) } for _, id := range indexed { err = i.Update(&document.Document{ ID: id, Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } } indexReader, err := i.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() searcher, err := NewDocIDSearcher(indexReader, searched, 1.0, false) if err != nil { t.Fatal(err) } defer func() { err := searcher.Close() if err != nil { t.Fatal(err) } }() if searcher.Count() != uint64(len(wanted)) { t.Fatalf("expected count %v got %v", len(wanted), searcher.Count()) } // Check the sequence for i, id := range wanted { m, err := searcher.Next() if err != nil { t.Fatal(err) } if id != m.ID { t.Fatalf("expected %v at position %v, got %v", id, i, m.ID) } } m, err := searcher.Next() if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("expected nil past the end of the sequence, got %v", m.ID) } // Check seeking for _, id := range wanted { if len(id) != 2 { t.Fatalf("expected identifier must be 2 characters long, got %v", id) } before := id[:1] for _, target := range []string{before, id} { m, err := searcher.Advance(target) if err != nil { t.Fatal(err) } if m == nil || m.ID != id { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } } } // Seek after the end of the sequence after := "zzz" m, err = searcher.Advance(after) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("advancing past the end of the sequence should return nil, got %v", m) } }
func openIndex(path string) (*indexImpl, error) { rv := indexImpl{ path: path, } var err error rv.meta, err = openIndexMeta(path) if err != nil { return nil, err } storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) if storeConstructor == nil { return nil, ErrorUnknownStorageType } storeConfig := map[string]interface{}{ "path": indexStorePath(path), "create_if_missing": false, "error_if_exists": false, } // now open the store rv.s, err = storeConstructor(storeConfig) if err != nil { return nil, err } // open open the index rv.i = upside_down.NewUpsideDownCouch(rv.s) err = rv.i.Open() if err != nil { return nil, err } // now load the mapping indexReader := rv.i.Reader() defer indexReader.Close() mappingBytes, err := indexReader.GetInternal(mappingInternalKey) if err != nil { return nil, err } var im IndexMapping err = json.Unmarshal(mappingBytes, &im) if err != nil { return nil, err } // mark the index as open rv.mutex.Lock() defer rv.mutex.Unlock() rv.open = true // validate the mapping err = im.validate() if err != nil { // note even if the mapping is invalid // we still return an open usable index return &rv, err } rv.m = &im return &rv, nil }
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { analysisQueue := index.NewAnalysisQueue(1) i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } err = i.Open() if err != nil { t.Fatal(err) } for _, id := range indexed { err = i.Update(&document.Document{ ID: id, Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } } indexReader, err := i.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() searcher, err := NewDocIDSearcher(indexReader, searched, 1.0, false) if err != nil { t.Fatal(err) } defer func() { err := searcher.Close() if err != nil { t.Fatal(err) } }() ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0), } // Check the sequence for i, id := range wanted { m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if !index.IndexInternalID(id).Equals(m.IndexInternalID) { t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID) } ctx.DocumentMatchPool.Put(m) } m, err := searcher.Next(ctx) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID) } ctx.DocumentMatchPool.Put(m) // Check seeking for _, id := range wanted { if len(id) != 2 { t.Fatalf("expected identifier must be 2 characters long, got %v", id) } before := id[:1] for _, target := range []string{before, id} { m, err := searcher.Advance(ctx, index.IndexInternalID(target)) if err != nil { t.Fatal(err) } if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) { t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) } ctx.DocumentMatchPool.Put(m) } } // Seek after the end of the sequence after := "zzz" m, err = searcher.Advance(ctx, index.IndexInternalID(after)) if err != nil { t.Fatal(err) } if m != nil { t.Fatalf("advancing past the end of the sequence should return nil, got %v", m) } ctx.DocumentMatchPool.Put(m) }
func TestTermSearcher(t *testing.T) { var queryTerm = "beer" var queryField = "desc" var queryBoost = 3.0 var queryExplain = true analysisQueue := index.NewAnalysisQueue(1) i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) if err != nil { t.Fatal(err) } err = i.Open() if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "a", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "b", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "c", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "d", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "e", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "f", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "g", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "h", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "i", Fields: []document.Field{ document.NewTextField("desc", []uint64{}, []byte("beer")), }, }) if err != nil { t.Fatal(err) } err = i.Update(&document.Document{ ID: "j", Fields: []document.Field{ document.NewTextField("title", []uint64{}, []byte("cat")), }, }) if err != nil { t.Fatal(err) } indexReader, err := i.Reader() if err != nil { t.Error(err) } defer func() { err := indexReader.Close() if err != nil { t.Fatal(err) } }() searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain) if err != nil { t.Fatal(err) } defer func() { err := searcher.Close() if err != nil { t.Fatal(err) } }() searcher.SetQueryNorm(2.0) docCount, err := indexReader.DocCount() if err != nil { t.Fatal(err) } idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0)) expectedQueryWeight := 3 * idf * 3 * idf if expectedQueryWeight != searcher.Weight() { t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight()) } if searcher.Count() != 9 { t.Errorf("expected count of 9, got %d", searcher.Count()) } ctx := &search.SearchContext{ DocumentMatchPool: search.NewDocumentMatchPool(1, 0), } docMatch, err := searcher.Next(ctx) if err != nil { t.Errorf("expected result, got %v", err) } if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) { t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID) } ctx.DocumentMatchPool.Put(docMatch) docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c")) if err != nil { t.Errorf("expected result, got %v", err) } if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) { t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID) } // try advancing past end ctx.DocumentMatchPool.Put(docMatch) docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z")) if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } // try pushing next past end ctx.DocumentMatchPool.Put(docMatch) docMatch, err = searcher.Next(ctx) if err != nil { t.Fatal(err) } if docMatch != nil { t.Errorf("expected nil, got %v", docMatch) } }