func constructQueryString(args []string) (url.Values, chunkStoreRecords) { qsValues := url.Values{} stores := chunkStoreRecords{} for _, arg := range args { k, v, ok := split2(arg, "=") if !ok { continue } // Magically assume that ldb: prefixed arguments are references to ldb stores. If so, construct // httpstore proxies to them, and rewrite the path to the client. // TODO: When clients can declare a nomdl interface, this can be much stricter. There should be // no need to search and attempt to string match every argument. if strings.HasPrefix(v, "ldb:") { _, path, _ := split2(v, ":") record, ok := stores[path] if !ok { record.cs = chunks.NewLevelDBStoreUseFlags(path, "") // Identify the stores with a (abridged) hash of the file system path, // so that the same URL always refers to the same database. hash := sha1.Sum([]byte(path)) record.alias = hex.EncodeToString(hash[:])[:8] stores[path] = record } v = fmt.Sprintf("%s/%s", dsPathPrefix, record.alias) } qsValues.Add(k, v) } return qsValues, stores }
func TestLDBDataset(t *testing.T) { assert := assert.New(t) dir, err := ioutil.TempDir(os.TempDir(), "") assert.NoError(err) ldbPath := path.Join(dir, "name") cs := chunks.NewLevelDBStoreUseFlags(ldbPath, "") ds := datas.NewDatabase(cs) id := "dsName" set := dataset.NewDataset(ds, id) commit := types.String("Commit Value") set, err = set.Commit(commit) assert.NoError(err) ds.Close() spec := fmt.Sprintf("ldb:%s::%s", ldbPath, id) sp, err := ParseDatasetSpec(spec) assert.NoError(err) dataset, err := sp.Dataset() assert.NoError(err) assert.EqualValues(commit, dataset.HeadValue()) os.Remove(dir) }
func TestLDBDatabase(t *testing.T) { assert := assert.New(t) d1 := os.TempDir() dir, err := ioutil.TempDir(d1, "flags") assert.NoError(err) ldbDir := path.Join(dir, "store") spec := fmt.Sprintf("ldb:%s", path.Join(dir, "store")) cs := chunks.NewLevelDBStoreUseFlags(ldbDir, "") ds := datas.NewDatabase(cs) s1 := types.String("A String") s1Ref := ds.WriteValue(s1) ds.Commit("testDs", datas.NewCommit().Set(datas.ValueField, s1Ref)) ds.Close() sp, errRead := ParseDatabaseSpec(spec) assert.NoError(errRead) store, err := sp.Database() assert.NoError(err) assert.Equal(s1, store.ReadValue(s1.Hash())) store.Close() os.Remove(dir) }
func TestReadRef(t *testing.T) { assert := assert.New(t) dir, err := ioutil.TempDir(os.TempDir(), "") assert.NoError(err) datasetId := "dsName" ldbPath := path.Join(dir, "/name") cs1 := chunks.NewLevelDBStoreUseFlags(ldbPath, "") database1 := datas.NewDatabase(cs1) dataset1 := dataset.NewDataset(database1, datasetId) commit := types.String("Commit Value") dataset1, err = dataset1.Commit(commit) assert.NoError(err) r1 := dataset1.Head().Hash() dataset1.Database().Close() spec2 := fmt.Sprintf("ldb:%s::%s", ldbPath, r1.String()) sp2, err := ParsePathSpec(spec2) assert.NoError(err) database, v2, err := sp2.Value() assert.NoError(err) assert.EqualValues(r1.String(), v2.Hash().String()) database.Close() }
func TestLDBObject(t *testing.T) { assert := assert.New(t) dir, err := ioutil.TempDir(os.TempDir(), "") assert.NoError(err) ldbpath := path.Join(dir, "xx-yy") dsId := "dsId" cs1 := chunks.NewLevelDBStoreUseFlags(ldbpath, "") store1 := datas.NewDatabase(cs1) dataset1 := dataset.NewDataset(store1, dsId) s1 := types.String("Commit Value") r1 := store1.WriteValue(s1) _, err = dataset1.Commit(r1) assert.NoError(err) store1.Close() spec2 := fmt.Sprintf("ldb:%s::%s", ldbpath, dsId) assert.NoError(err) sp1, err := ParseDatasetSpec(spec2) assert.NoError(err) dataset2, err := sp1.Dataset() assert.NoError(err) r2 := dataset2.HeadValue() s2 := r2.(types.Ref).TargetValue(dataset2.Database()) assert.Equal(s1, s2) dataset2.Database().Close() spec3 := fmt.Sprintf("ldb:%s::%s", ldbpath, s1.Hash().String()) sp3, err := ParsePathSpec(spec3) database, v3, err := sp3.Value() assert.Equal(s1, v3) database.Close() }
func (spec DatabaseSpec) ChunkStore() (cs chunks.ChunkStore, err error) { switch spec.Protocol { case "ldb": cs = chunks.NewLevelDBStoreUseFlags(spec.Path, "") case "mem": cs = chunks.NewMemoryStore() default: return nil, fmt.Errorf("Unable to create chunkstore for protocol: %s", spec) } return }
func (spec DatabaseSpec) Database() (ds datas.Database, err error) { switch spec.Protocol { case "http", "https": ds = datas.NewRemoteDatabase(spec.String(), "Bearer "+spec.accessToken) case "ldb": ds = datas.NewDatabase(chunks.NewLevelDBStoreUseFlags(spec.Path, "")) case "mem": ds = datas.NewDatabase(chunks.NewMemoryStore()) default: err = fmt.Errorf("Invalid path prototocol: %s", spec.Protocol) } return }
func newRefCountingLdbStore(path string, closeFn func()) *refCountingLdbStore { return &refCountingLdbStore{chunks.NewLevelDBStoreUseFlags(path, ""), 1, closeFn} }