func constructQueryString(args []string) (url.Values, chunkStoreRecords) {
qsValues := url.Values{}
stores := chunkStoreRecords{}
for _, arg := range args {
k, v, ok := split2(arg, "=")
if !ok {
continue
}
// Magically assume that ldb: prefixed arguments are references to ldb stores. If so, construct
// httpstore proxies to them, and rewrite the path to the client.
// TODO: When clients can declare a nomdl interface, this can be much stricter. There should be
// no need to search and attempt to string match every argument.
if strings.HasPrefix(v, "ldb:") {
_, path, _ := split2(v, ":")
record, ok := stores[path]
if !ok {
record.cs = chunks.NewLevelDBStoreUseFlags(path, "")
// Identify the stores with a (abridged) hash of the file system path,
// so that the same URL always refers to the same database.
hash := sha1.Sum([]byte(path))
record.alias = hex.EncodeToString(hash[:])[:8]
stores[path] = record
}
v = fmt.Sprintf("%s/%s", dsPathPrefix, record.alias)
}
qsValues.Add(k, v)
}
return qsValues, stores
}
func TestLDBDataset(t *testing.T) {
assert := assert.New(t)
dir, err := ioutil.TempDir(os.TempDir(), "")
assert.NoError(err)
ldbPath := path.Join(dir, "name")
cs := chunks.NewLevelDBStoreUseFlags(ldbPath, "")
ds := datas.NewDatabase(cs)
id := "dsName"
set := dataset.NewDataset(ds, id)
commit := types.String("Commit Value")
set, err = set.Commit(commit)
assert.NoError(err)
ds.Close()
spec := fmt.Sprintf("ldb:%s::%s", ldbPath, id)
sp, err := ParseDatasetSpec(spec)
assert.NoError(err)
dataset, err := sp.Dataset()
assert.NoError(err)
assert.EqualValues(commit, dataset.HeadValue())
os.Remove(dir)
}
func TestLDBDatabase(t *testing.T) {
assert := assert.New(t)
d1 := os.TempDir()
dir, err := ioutil.TempDir(d1, "flags")
assert.NoError(err)
ldbDir := path.Join(dir, "store")
spec := fmt.Sprintf("ldb:%s", path.Join(dir, "store"))
cs := chunks.NewLevelDBStoreUseFlags(ldbDir, "")
ds := datas.NewDatabase(cs)
s1 := types.String("A String")
s1Ref := ds.WriteValue(s1)
ds.Commit("testDs", datas.NewCommit().Set(datas.ValueField, s1Ref))
ds.Close()
sp, errRead := ParseDatabaseSpec(spec)
assert.NoError(errRead)
store, err := sp.Database()
assert.NoError(err)
assert.Equal(s1, store.ReadValue(s1.Hash()))
store.Close()
os.Remove(dir)
}
func TestReadRef(t *testing.T) {
assert := assert.New(t)
dir, err := ioutil.TempDir(os.TempDir(), "")
assert.NoError(err)
datasetId := "dsName"
ldbPath := path.Join(dir, "/name")
cs1 := chunks.NewLevelDBStoreUseFlags(ldbPath, "")
database1 := datas.NewDatabase(cs1)
dataset1 := dataset.NewDataset(database1, datasetId)
commit := types.String("Commit Value")
dataset1, err = dataset1.Commit(commit)
assert.NoError(err)
r1 := dataset1.Head().Hash()
dataset1.Database().Close()
spec2 := fmt.Sprintf("ldb:%s::%s", ldbPath, r1.String())
sp2, err := ParsePathSpec(spec2)
assert.NoError(err)
database, v2, err := sp2.Value()
assert.NoError(err)
assert.EqualValues(r1.String(), v2.Hash().String())
database.Close()
}
func TestLDBObject(t *testing.T) {
assert := assert.New(t)
dir, err := ioutil.TempDir(os.TempDir(), "")
assert.NoError(err)
ldbpath := path.Join(dir, "xx-yy")
dsId := "dsId"
cs1 := chunks.NewLevelDBStoreUseFlags(ldbpath, "")
store1 := datas.NewDatabase(cs1)
dataset1 := dataset.NewDataset(store1, dsId)
s1 := types.String("Commit Value")
r1 := store1.WriteValue(s1)
_, err = dataset1.Commit(r1)
assert.NoError(err)
store1.Close()
spec2 := fmt.Sprintf("ldb:%s::%s", ldbpath, dsId)
assert.NoError(err)
sp1, err := ParseDatasetSpec(spec2)
assert.NoError(err)
dataset2, err := sp1.Dataset()
assert.NoError(err)
r2 := dataset2.HeadValue()
s2 := r2.(types.Ref).TargetValue(dataset2.Database())
assert.Equal(s1, s2)
dataset2.Database().Close()
spec3 := fmt.Sprintf("ldb:%s::%s", ldbpath, s1.Hash().String())
sp3, err := ParsePathSpec(spec3)
database, v3, err := sp3.Value()
assert.Equal(s1, v3)
database.Close()
}
func (spec DatabaseSpec) ChunkStore() (cs chunks.ChunkStore, err error) {
switch spec.Protocol {
case "ldb":
cs = chunks.NewLevelDBStoreUseFlags(spec.Path, "")
case "mem":
cs = chunks.NewMemoryStore()
default:
return nil, fmt.Errorf("Unable to create chunkstore for protocol: %s", spec)
}
return
}
func (spec DatabaseSpec) Database() (ds datas.Database, err error) {
switch spec.Protocol {
case "http", "https":
ds = datas.NewRemoteDatabase(spec.String(), "Bearer "+spec.accessToken)
case "ldb":
ds = datas.NewDatabase(chunks.NewLevelDBStoreUseFlags(spec.Path, ""))
case "mem":
ds = datas.NewDatabase(chunks.NewMemoryStore())
default:
err = fmt.Errorf("Invalid path prototocol: %s", spec.Protocol)
}
return
}
func newRefCountingLdbStore(path string, closeFn func()) *refCountingLdbStore {
return &refCountingLdbStore{chunks.NewLevelDBStoreUseFlags(path, ""), 1, closeFn}
}
