Beispiel #1
0
func constructQueryString(args []string) (url.Values, chunkStoreRecords) {
	qsValues := url.Values{}
	stores := chunkStoreRecords{}

	for _, arg := range args {
		k, v, ok := split2(arg, "=")
		if !ok {
			continue
		}

		// Magically assume that ldb: prefixed arguments are references to ldb stores. If so, construct
		// httpstore proxies to them, and rewrite the path to the client.
		// TODO: When clients can declare a nomdl interface, this can be much stricter. There should be
		// no need to search and attempt to string match every argument.
		if strings.HasPrefix(v, "ldb:") {
			_, path, _ := split2(v, ":")
			record, ok := stores[path]
			if !ok {
				record.cs = chunks.NewLevelDBStoreUseFlags(path, "")
				// Identify the stores with a (abridged) hash of the file system path,
				// so that the same URL always refers to the same database.
				hash := sha1.Sum([]byte(path))
				record.alias = hex.EncodeToString(hash[:])[:8]
				stores[path] = record
			}
			v = fmt.Sprintf("%s/%s", dsPathPrefix, record.alias)
		}

		qsValues.Add(k, v)
	}

	return qsValues, stores
}
Beispiel #2
0
func TestLDBDataset(t *testing.T) {
	assert := assert.New(t)

	dir, err := ioutil.TempDir(os.TempDir(), "")
	assert.NoError(err)
	ldbPath := path.Join(dir, "name")
	cs := chunks.NewLevelDBStoreUseFlags(ldbPath, "")
	ds := datas.NewDatabase(cs)
	id := "dsName"

	set := dataset.NewDataset(ds, id)
	commit := types.String("Commit Value")
	set, err = set.Commit(commit)
	assert.NoError(err)
	ds.Close()

	spec := fmt.Sprintf("ldb:%s::%s", ldbPath, id)
	sp, err := ParseDatasetSpec(spec)
	assert.NoError(err)
	dataset, err := sp.Dataset()
	assert.NoError(err)
	assert.EqualValues(commit, dataset.HeadValue())

	os.Remove(dir)
}
Beispiel #3
0
func TestLDBDatabase(t *testing.T) {
	assert := assert.New(t)

	d1 := os.TempDir()
	dir, err := ioutil.TempDir(d1, "flags")
	assert.NoError(err)
	ldbDir := path.Join(dir, "store")
	spec := fmt.Sprintf("ldb:%s", path.Join(dir, "store"))

	cs := chunks.NewLevelDBStoreUseFlags(ldbDir, "")
	ds := datas.NewDatabase(cs)

	s1 := types.String("A String")
	s1Ref := ds.WriteValue(s1)
	ds.Commit("testDs", datas.NewCommit().Set(datas.ValueField, s1Ref))
	ds.Close()

	sp, errRead := ParseDatabaseSpec(spec)
	assert.NoError(errRead)
	store, err := sp.Database()
	assert.NoError(err)
	assert.Equal(s1, store.ReadValue(s1.Hash()))
	store.Close()
	os.Remove(dir)
}
Beispiel #4
0
func TestReadRef(t *testing.T) {
	assert := assert.New(t)

	dir, err := ioutil.TempDir(os.TempDir(), "")
	assert.NoError(err)
	datasetId := "dsName"

	ldbPath := path.Join(dir, "/name")
	cs1 := chunks.NewLevelDBStoreUseFlags(ldbPath, "")
	database1 := datas.NewDatabase(cs1)
	dataset1 := dataset.NewDataset(database1, datasetId)
	commit := types.String("Commit Value")
	dataset1, err = dataset1.Commit(commit)
	assert.NoError(err)
	r1 := dataset1.Head().Hash()
	dataset1.Database().Close()

	spec2 := fmt.Sprintf("ldb:%s::%s", ldbPath, r1.String())
	sp2, err := ParsePathSpec(spec2)
	assert.NoError(err)
	database, v2, err := sp2.Value()
	assert.NoError(err)

	assert.EqualValues(r1.String(), v2.Hash().String())
	database.Close()
}
Beispiel #5
0
func TestLDBObject(t *testing.T) {
	assert := assert.New(t)
	dir, err := ioutil.TempDir(os.TempDir(), "")
	assert.NoError(err)
	ldbpath := path.Join(dir, "xx-yy")
	dsId := "dsId"

	cs1 := chunks.NewLevelDBStoreUseFlags(ldbpath, "")
	store1 := datas.NewDatabase(cs1)
	dataset1 := dataset.NewDataset(store1, dsId)
	s1 := types.String("Commit Value")
	r1 := store1.WriteValue(s1)
	_, err = dataset1.Commit(r1)
	assert.NoError(err)
	store1.Close()

	spec2 := fmt.Sprintf("ldb:%s::%s", ldbpath, dsId)
	assert.NoError(err)
	sp1, err := ParseDatasetSpec(spec2)
	assert.NoError(err)
	dataset2, err := sp1.Dataset()
	assert.NoError(err)
	r2 := dataset2.HeadValue()
	s2 := r2.(types.Ref).TargetValue(dataset2.Database())
	assert.Equal(s1, s2)
	dataset2.Database().Close()

	spec3 := fmt.Sprintf("ldb:%s::%s", ldbpath, s1.Hash().String())
	sp3, err := ParsePathSpec(spec3)
	database, v3, err := sp3.Value()
	assert.Equal(s1, v3)
	database.Close()
}
Beispiel #6
0
func (spec DatabaseSpec) ChunkStore() (cs chunks.ChunkStore, err error) {
	switch spec.Protocol {
	case "ldb":
		cs = chunks.NewLevelDBStoreUseFlags(spec.Path, "")
	case "mem":
		cs = chunks.NewMemoryStore()
	default:
		return nil, fmt.Errorf("Unable to create chunkstore for protocol: %s", spec)
	}

	return
}
Beispiel #7
0
func (spec DatabaseSpec) Database() (ds datas.Database, err error) {
	switch spec.Protocol {
	case "http", "https":
		ds = datas.NewRemoteDatabase(spec.String(), "Bearer "+spec.accessToken)
	case "ldb":
		ds = datas.NewDatabase(chunks.NewLevelDBStoreUseFlags(spec.Path, ""))
	case "mem":
		ds = datas.NewDatabase(chunks.NewMemoryStore())
	default:
		err = fmt.Errorf("Invalid path prototocol: %s", spec.Protocol)
	}
	return
}
func newRefCountingLdbStore(path string, closeFn func()) *refCountingLdbStore {
	return &refCountingLdbStore{chunks.NewLevelDBStoreUseFlags(path, ""), 1, closeFn}
}