func newIndex(path string, mapping *IndexMapping) (*indexImpl, error) {
// first validate the mapping
err := mapping.validate()
if err != nil {
return nil, err
}
if path == "" {
return newMemIndex(mapping)
}
rv := indexImpl{
path: path,
m: mapping,
meta: newIndexMeta(Config.DefaultKVStore),
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// at this point there hope we can be successful, so save index meta
err = rv.meta.Save(path)
if err != nil {
return nil, err
}
storeConfig := map[string]interface{}{
"path": indexStorePath(path),
"create_if_missing": true,
"error_if_exists": true,
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func NewBlevePIndexImpl(indexType, indexParams, path string,
restart func()) (cbgt.PIndexImpl, cbgt.Dest, error) {
bleveParams := NewBleveParams()
if len(indexParams) > 0 {
err := json.Unmarshal([]byte(indexParams), bleveParams)
if err != nil {
return nil, nil, fmt.Errorf("bleve: parse params, err: %v", err)
}
}
kvStoreName, ok := bleveParams.Store["kvStoreName"].(string)
if !ok || kvStoreName == "" {
kvStoreName = bleve.Config.DefaultKVStore
}
kvConfig := map[string]interface{}{
"create_if_missing": true,
"error_if_exists": true,
}
for k, v := range bleveParams.Store {
kvConfig[k] = v
}
// Always use the "metrics" wrapper KVStore if it's available and
// also not already configured.
_, exists := kvConfig["kvStoreName_actual"]
if !exists &&
kvStoreName != "metrics" &&
bleveRegistry.KVStoreConstructorByName("metrics") != nil {
kvConfig["kvStoreName_actual"] = kvStoreName
kvStoreName = "metrics"
}
bleveIndexType, ok := bleveParams.Store["indexType"].(string)
if !ok || bleveIndexType == "" {
bleveIndexType = bleve.Config.DefaultIndexType
}
bindex, err := bleve.NewUsing(path, &bleveParams.Mapping,
bleveIndexType, kvStoreName, kvConfig)
if err != nil {
return nil, nil, fmt.Errorf("bleve: new index, path: %s,"+
" kvStoreName: %s, kvConfig: %#v, err: %s",
path, kvStoreName, kvConfig, err)
}
pathMeta := path + string(os.PathSeparator) + "PINDEX_BLEVE_META"
err = ioutil.WriteFile(pathMeta, []byte(indexParams), 0600)
if err != nil {
return nil, nil, err
}
return bindex, &cbgt.DestForwarder{
DestProvider: NewBleveDest(path, bindex, restart),
}, nil
}
func initLowerLevelStore(
config map[string]interface{},
lowerLevelStoreName string,
lowerLevelStoreConfig map[string]interface{},
lowerLevelMaxBatchSize uint64,
options moss.CollectionOptions,
) (moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, error) {
if lowerLevelStoreConfig == nil {
lowerLevelStoreConfig = map[string]interface{}{}
}
for k, v := range config {
_, exists := lowerLevelStoreConfig[k]
if !exists {
lowerLevelStoreConfig[k] = v
}
}
if lowerLevelStoreName == "mossStore" {
return InitMossStore(lowerLevelStoreConfig, options)
}
constructor := registry.KVStoreConstructorByName(lowerLevelStoreName)
if constructor == nil {
return nil, nil, nil, fmt.Errorf("moss store, initLowerLevelStore,"+
" could not find lower level store: %s", lowerLevelStoreName)
}
kvStore, err := constructor(options.MergeOperator, lowerLevelStoreConfig)
if err != nil {
return nil, nil, nil, err
}
llStore := &llStore{
refs: 0,
config: config,
llConfig: lowerLevelStoreConfig,
kvStore: kvStore,
logf: options.Log,
}
llUpdate := func(ssHigher moss.Snapshot) (ssLower moss.Snapshot, err error) {
return llStore.update(ssHigher, lowerLevelMaxBatchSize)
}
llSnapshot, err := llUpdate(nil)
if err != nil {
_ = kvStore.Close()
return nil, nil, nil, err
}
return llSnapshot, llUpdate, kvStore, nil // llStore.refs is now 1.
}
func newMemIndex(indexType string, mapping *IndexMapping) (*indexImpl, error) {
rv := indexImpl{
path: "",
m: mapping,
meta: newIndexMeta(indexType, "mem", nil),
stats: &IndexStat{},
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// now open the store
var err error
rv.s, err = storeConstructor(nil)
if err != nil {
return nil, err
}
// open the index
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
if indexTypeConstructor == nil {
return nil, ErrorUnknownIndexType
}
rv.i, err = indexTypeConstructor(rv.s, Config.analysisQueue)
if err != nil {
return nil, err
}
err = rv.i.Open()
if err != nil {
return nil, err
}
rv.stats.indexStat = rv.i.Stats()
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
name, ok := config["kvStoreName_actual"].(string)
if !ok || name == "" {
return nil, fmt.Errorf("preload: missing kvStoreName_actual,"+
" config: %#v", config)
}
if name == Name {
return nil, fmt.Errorf("preload: circular kvStoreName_actual")
}
ctr := registry.KVStoreConstructorByName(name)
if ctr == nil {
return nil, fmt.Errorf("preload: no kv store constructor,"+
" kvStoreName_actual: %s", name)
}
kvs, err := ctr(mo, config)
if err != nil {
return nil, err
}
rv := &Store{
o: kvs,
}
if preloadPath, ok := config["preloadpath"].(string); ok {
f, err := os.Open(preloadPath)
if err != nil {
return nil, err
}
gzr, err := gzip.NewReader(f)
if err != nil {
return nil, err
}
err = Import(rv, gzr, 1024)
if err != nil {
return nil, err
}
err = gzr.Close()
if err != nil {
return nil, err
}
err = f.Close()
if err != nil {
return nil, err
}
}
return rv, nil
}
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
name, ok := config["kvStoreName_actual"].(string)
if !ok || name == "" {
return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
" config: %#v", config)
}
if name == Name {
return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
}
ctr := registry.KVStoreConstructorByName(name)
if ctr == nil {
return nil, fmt.Errorf("metrics: no kv store constructor,"+
" kvStoreName_actual: %s", name)
}
kvs, err := ctr(mo, config)
if err != nil {
return nil, err
}
rv := &Store{
o: kvs,
timerReaderGet: metrics.NewTimer(),
timerReaderMultiGet: metrics.NewTimer(),
timerReaderPrefixIterator: metrics.NewTimer(),
timerReaderRangeIterator: metrics.NewTimer(),
timerWriterExecuteBatch: metrics.NewTimer(),
timerIteratorSeek: metrics.NewTimer(),
timerIteratorNext: metrics.NewTimer(),
timerBatchMerge: metrics.NewTimer(),
errors: list.New(),
}
rv.s = &stats{s: rv}
return rv, nil
}
func newMemIndex(mapping *IndexMapping) (*indexImpl, error) {
rv := indexImpl{
path: "",
m: mapping,
meta: newIndexMeta("mem"),
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// now open the store
var err error
rv.s, err = storeConstructor(nil)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
name, ok := config["kvStoreName_actual"].(string)
if !ok || name == "" {
return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
" config: %#v", config)
}
if name == Name {
return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
}
ctr := registry.KVStoreConstructorByName(name)
if ctr == nil {
return nil, fmt.Errorf("metrics: no kv store constructor,"+
" kvStoreName_actual: %s", name)
}
kvs, err := ctr(config)
if err != nil {
return nil, err
}
return NewBleveMetricsStore(kvs), nil
}
func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
// first validate the mapping
err := mapping.validate()
if err != nil {
return nil, err
}
if path == "" {
return newMemIndex(indexType, mapping)
}
if kvconfig == nil {
kvconfig = map[string]interface{}{}
}
rv := indexImpl{
path: path,
m: mapping,
meta: newIndexMeta(indexType, kvstore, kvconfig),
stats: &IndexStat{},
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
// at this point there is hope that we can be successful, so save index meta
err = rv.meta.Save(path)
if err != nil {
return nil, err
}
kvconfig["create_if_missing"] = true
kvconfig["error_if_exists"] = true
kvconfig["path"] = indexStorePath(path)
// now create the store
rv.s, err = storeConstructor(kvconfig)
if err != nil {
return nil, err
}
// open the index
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
if indexTypeConstructor == nil {
return nil, ErrorUnknownIndexType
}
rv.i, err = indexTypeConstructor(rv.s, Config.analysisQueue)
if err != nil {
return nil, err
}
err = rv.i.Open()
if err != nil {
return nil, err
}
rv.stats.indexStat = rv.i.Stats()
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
return &rv, nil
}
func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) {
rv = &indexImpl{
path: path,
stats: &IndexStat{},
}
rv.meta, err = openIndexMeta(path)
if err != nil {
return nil, err
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
storeConfig := rv.meta.Config
if storeConfig == nil {
storeConfig = map[string]interface{}{}
}
storeConfig["path"] = indexStorePath(path)
storeConfig["create_if_missing"] = false
storeConfig["error_if_exists"] = false
for rck, rcv := range runtimeConfig {
storeConfig[rck] = rcv
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open the index
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
if indexTypeConstructor == nil {
return nil, ErrorUnknownIndexType
}
rv.i, err = indexTypeConstructor(rv.s, Config.analysisQueue)
if err != nil {
return nil, err
}
err = rv.i.Open()
if err != nil {
return nil, err
}
rv.stats.indexStat = rv.i.Stats()
// now load the mapping
indexReader, err := rv.i.Reader()
if err != nil {
return nil, err
}
defer func() {
if cerr := indexReader.Close(); cerr != nil && err == nil {
err = cerr
}
}()
mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
if err != nil {
return nil, err
}
var im IndexMapping
err = json.Unmarshal(mappingBytes, &im)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
// validate the mapping
err = im.validate()
if err != nil {
// note even if the mapping is invalid
// we still return an open usable index
return rv, err
}
rv.m = &im
return rv, err
}
func (f *Firestorm) Open() (err error) {
// open the kv store
storeConstructor := registry.KVStoreConstructorByName(f.storeName)
if storeConstructor == nil {
err = index.ErrorUnknownStorageType
return
}
// now open the store
f.store, err = storeConstructor(&mergeOperator, f.storeConfig)
if err != nil {
return
}
// start a reader
var kvreader store.KVReader
kvreader, err = f.store.Reader()
if err != nil {
return
}
// assert correct version, and find out if this is new index
var newIndex bool
newIndex, err = f.checkVersion(kvreader)
if err != nil {
return
}
if !newIndex {
// process existing index before opening
err = f.warmup(kvreader)
if err != nil {
return
}
}
err = kvreader.Close()
if err != nil {
return
}
if newIndex {
// prepare a new index
err = f.bootstrap()
if err != nil {
return
}
}
// start the garbage collector
f.garbageCollector.Start()
// start the lookuper
f.lookuper.Start()
// start the dict updater
f.dictUpdater.Start()
return
}
func openIndex(path string) (*indexImpl, error) {
rv := indexImpl{
path: path,
}
var err error
rv.meta, err = openIndexMeta(path)
if err != nil {
return nil, err
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ErrorUnknownStorageType
}
storeConfig := map[string]interface{}{
"path": indexStorePath(path),
"create_if_missing": false,
"error_if_exists": false,
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now load the mapping
indexReader := rv.i.Reader()
defer indexReader.Close()
mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
if err != nil {
return nil, err
}
var im IndexMapping
err = json.Unmarshal(mappingBytes, &im)
if err != nil {
return nil, err
}
// mark the index as open
rv.mutex.Lock()
defer rv.mutex.Unlock()
rv.open = true
// validate the mapping
err = im.validate()
if err != nil {
// note even if the mapping is invalid
// we still return an open usable index
return &rv, err
}
rv.m = &im
return &rv, nil
}
func (udc *SmolderingCouch) Open() (err error) {
//acquire the write mutex for the duratin of Open()
udc.writeMutex.Lock()
defer udc.writeMutex.Unlock()
// open the kv store
storeConstructor := registry.KVStoreConstructorByName(udc.storeName)
if storeConstructor == nil {
err = index.ErrorUnknownStorageType
return
}
// now open the store
udc.store, err = storeConstructor(&mergeOperator, udc.storeConfig)
if err != nil {
return
}
udc.cf = cuckoofilter.NewDefaultCuckooFilter()
// start a reader to look at the index
var kvreader store.KVReader
kvreader, err = udc.store.Reader()
if err != nil {
return
}
var value []byte
value, err = kvreader.Get(VersionKey)
if err != nil {
_ = kvreader.Close()
return
}
if value != nil {
err = udc.loadSchema(kvreader)
if err != nil {
_ = kvreader.Close()
return
}
// set doc count
udc.m.Lock()
udc.docCount, udc.maxInternalDocID, err = udc.countDocs(kvreader)
udc.m.Unlock()
err = kvreader.Close()
} else {
// new index, close the reader and open writer to init
err = kvreader.Close()
if err != nil {
return
}
var kvwriter store.KVWriter
kvwriter, err = udc.store.Writer()
if err != nil {
return
}
defer func() {
if cerr := kvwriter.Close(); err == nil && cerr != nil {
err = cerr
}
}()
// init the index
err = udc.init(kvwriter)
}
return
}