Example #1
0
func TestCoalesceSamePutDiffPut(t *testing.T) {
	m := setup()
	done := make(chan struct{})

	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar")
		done <- struct{}{}
	}()
	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar")
		done <- struct{}{}
	}()
	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar2")
		done <- struct{}{}
	}()
	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar3")
		done <- struct{}{}
	}()

	<-done
	<-done
	<-done
	<-done

	if m.inside != 3 {
		t.Error("incalls should be 3", m.inside)
	}

	if m.outside != 4 {
		t.Error("outcalls should be 4", m.outside)
	}
}
Example #2
0
func (ks *DSSuite) TestBasic(c *C) {
	var size = 1000

	d, err := lru.NewDatastore(size)
	c.Check(err, Equals, nil)

	for i := 0; i < size; i++ {
		err := d.Put(ds.NewKey(strconv.Itoa(i)), i)
		c.Check(err, Equals, nil)
	}

	for i := 0; i < size; i++ {
		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
		c.Check(j, Equals, i)
		c.Check(err, Equals, nil)
	}

	for i := 0; i < size; i++ {
		err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i)
		c.Check(err, Equals, nil)
	}

	for i := 0; i < size; i++ {
		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
		c.Check(j, Equals, nil)
		c.Check(err, Equals, ds.ErrNotFound)
	}

	for i := 0; i < size; i++ {
		j, err := d.Get(ds.NewKey(strconv.Itoa(i + size)))
		c.Check(j, Equals, i)
		c.Check(err, Equals, nil)
	}
}
Example #3
0
func TestCoalesceHas(t *testing.T) {
	m := setup()
	done := make(chan struct{})
	errs := make(chan error, 30)

	m.ds.Put(ds.NewKey("foo1"), "bar")
	m.ds.Put(ds.NewKey("foo2"), "baz")

	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Has(ds.NewKey("foo1"))
			if err != nil {
				errs <- err
			}
			if !v {
				errs <- fmt.Errorf("should have foo1")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Has(ds.NewKey("foo2"))
			if err != nil {
				errs <- err
			}
			if !v {
				errs <- fmt.Errorf("should have foo2")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Has(ds.NewKey("foo3"))
			if err != nil {
				errs <- err
			}
			if v {
				errs <- fmt.Errorf("should not have foo3")
			}
			done <- struct{}{}
		}()
	}

	for i := 0; i < 30; i++ {
		<-done
	}

	if m.inside != 5 {
		t.Error("incalls should be 3", m.inside)
	}

	if m.outside != 32 {
		t.Error("outcalls should be 30", m.outside)
	}
}
Example #4
0
// openDatastore returns an error if the config file is not present.
func (r *FSRepo) openDatastore() error {
	leveldbPath := path.Join(r.path, leveldbDirectory)
	var err error
	// save leveldb reference so it can be neatly closed afterward
	r.leveldbDS, err = levelds.NewDatastore(leveldbPath, &levelds.Options{
		Compression: ldbopts.NoCompression,
	})
	if err != nil {
		return errors.New("unable to open leveldb datastore")
	}

	// 4TB of 256kB objects ~=17M objects, splitting that 256-way
	// leads to ~66k objects per dir, splitting 256*256-way leads to
	// only 256.
	//
	// The keys seen by the block store have predictable prefixes,
	// including "/" from datastore.Key and 2 bytes from multihash. To
	// reach a uniform 256-way split, we need approximately 4 bytes of
	// prefix.
	blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
	if err != nil {
		return errors.New("unable to open flatfs datastore")
	}

	// Add our PeerID to metrics paths to keep them unique
	//
	// As some tests just pass a zero-value Config to fsrepo.Init,
	// cope with missing PeerID.
	id := r.config.Identity.PeerID
	if id == "" {
		// the tests pass in a zero Config; cope with it
		id = fmt.Sprintf("uninitialized_%p", r)
	}
	prefix := "fsrepo." + id + ".datastore."
	r.metricsBlocks = measure.New(prefix+"blocks", blocksDS)
	r.metricsLevelDB = measure.New(prefix+"leveldb", r.leveldbDS)
	mountDS := mount.New([]mount.Mount{
		{
			Prefix:    ds.NewKey("/blocks"),
			Datastore: r.metricsBlocks,
		},
		{
			Prefix:    ds.NewKey("/"),
			Datastore: r.metricsLevelDB,
		},
	})
	// Make sure it's ok to claim the virtual datastore from mount as
	// threadsafe. There's no clean way to make mount itself provide
	// this information without copy-pasting the code into two
	// variants. This is the same dilemma as the `[].byte` attempt at
	// introducing const types to Go.
	var _ ds.ThreadSafeDatastore = blocksDS
	var _ ds.ThreadSafeDatastore = r.leveldbDS
	r.ds = ds2.ClaimThreadSafe{mountDS}
	return nil
}
Example #5
0
func TestCoalesceSameGet(t *testing.T) {
	m := setup()
	done := make(chan struct{})
	errs := make(chan error, 30)

	m.ds.Put(ds.NewKey("foo1"), "bar")
	m.ds.Put(ds.NewKey("foo2"), "baz")

	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Get(ds.NewKey("foo1"))
			if err != nil {
				errs <- err
			}
			if v != "bar" {
				errs <- fmt.Errorf("v is not bar", v)
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Get(ds.NewKey("foo2"))
			if err != nil {
				errs <- err
			}
			if v != "baz" {
				errs <- fmt.Errorf("v is not baz", v)
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			_, err := m.ds.Get(ds.NewKey("foo3"))
			if err == nil {
				errs <- fmt.Errorf("no error")
			}
			done <- struct{}{}
		}()
	}

	for i := 0; i < 30; i++ {
		<-done
	}

	if m.inside != 5 {
		t.Error("incalls should be 3", m.inside)
	}

	if m.outside != 32 {
		t.Error("outcalls should be 30", m.outside)
	}
}
Example #6
0
func (d *Datastore) lookup(key ds.Key) (ds.Datastore, ds.Key, ds.Key) {
	d.lk.Lock()
	defer d.lk.Unlock()
	for _, m := range d.mounts {
		if m.Prefix.Equal(key) || m.Prefix.IsAncestorOf(key) {
			s := strings.TrimPrefix(key.String(), m.Prefix.String())
			k := ds.NewKey(s)
			return m.Datastore, m.Prefix, k
		}
	}
	return nil, ds.NewKey("/"), key
}
Example #7
0
func (ks *DSSuite) TestBasic(c *C) {

	mpds := ds.NewMapDatastore()
	nsds := ns.Wrap(mpds, ds.NewKey("abc"))

	keys := strsToKeys([]string{
		"foo",
		"foo/bar",
		"foo/bar/baz",
		"foo/barb",
		"foo/bar/bazb",
		"foo/bar/baz/barb",
	})

	for _, k := range keys {
		err := nsds.Put(k, []byte(k.String()))
		c.Check(err, Equals, nil)
	}

	for _, k := range keys {
		v1, err := nsds.Get(k)
		c.Check(err, Equals, nil)
		c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)

		v2, err := mpds.Get(ds.NewKey("abc").Child(k))
		c.Check(err, Equals, nil)
		c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
	}

	run := func(d ds.Datastore, q dsq.Query) []ds.Key {
		r, err := d.Query(q)
		c.Check(err, Equals, nil)

		e, err := r.Rest()
		c.Check(err, Equals, nil)

		return ds.EntryKeys(e)
	}

	listA := run(mpds, dsq.Query{})
	listB := run(nsds, dsq.Query{})
	c.Check(len(listA), Equals, len(listB))

	// sort them cause yeah.
	sort.Sort(ds.KeySlice(listA))
	sort.Sort(ds.KeySlice(listB))

	for i, kA := range listA {
		kB := listB[i]
		c.Check(nsds.InvertKey(kA), Equals, kB)
		c.Check(kA, Equals, nsds.ConvertKey(kB))
	}
}
Example #8
0
// Query implements Query, inverting keys on the way back out.
func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
	qr, err := d.raw.Query(q)
	if err != nil {
		return nil, err
	}

	ch := make(chan dsq.Result)
	go func() {
		defer close(ch)
		defer qr.Close()

		for r := range qr.Next() {
			if r.Error != nil {
				ch <- r
				continue
			}

			k := ds.NewKey(r.Entry.Key)
			if !d.prefix.IsAncestorOf(k) {
				continue
			}

			r.Entry.Key = d.Datastore.InvertKey(k).String()
			ch <- r
		}
	}()

	return dsq.DerivedResults(qr, ch), nil
}
Example #9
0
// Query implements Datastore.Query
func (d *Datastore) Query(q query.Query) (query.Results, error) {

	results := make(chan query.Result)

	walkFn := func(path string, info os.FileInfo, err error) error {
		// remove ds path prefix
		if strings.HasPrefix(path, d.path) {
			path = path[len(d.path):]
		}

		if !info.IsDir() {
			if strings.HasSuffix(path, ObjectKeySuffix) {
				path = path[:len(path)-len(ObjectKeySuffix)]
			}
			key := ds.NewKey(path)
			entry := query.Entry{Key: key.String(), Value: query.NotFetched}
			results <- query.Result{Entry: entry}
		}
		return nil
	}

	go func() {
		filepath.Walk(d.path, walkFn)
		close(results)
	}()
	r := query.ResultsWithChan(q, results)
	r = query.NaiveQueryApply(q, r)
	return r, nil
}
Example #10
0
func strsToKeys(strs []string) []ds.Key {
	keys := make([]ds.Key, len(strs))
	for i, s := range strs {
		keys[i] = ds.NewKey(s)
	}
	return keys
}
Example #11
0
// ConvertKey returns a B58 encoded Datastore key
// TODO: this is hacky because it encodes every path component. some
// path components may be proper strings already...
func (b58KeyConverter) ConvertKey(dsk ds.Key) ds.Key {
	k := ds.NewKey("/")
	for _, n := range dsk.Namespaces() {
		k = k.ChildString(b58.Encode([]byte(n)))
	}
	return k
}
Example #12
0
func TestDatastoreGetNotAllowedAfterClose(t *testing.T) {
	t.Parallel()
	path := testRepoPath("test", t)

	assert.True(!IsInitialized(path), t, "should NOT be initialized")
	assert.Nil(Init(path, &config.Config{}), t, "should initialize successfully")
	r, err := Open(path)
	assert.Nil(err, t, "should open successfully")

	k := "key"
	data := []byte(k)
	assert.Nil(r.Datastore().Put(datastore.NewKey(k), data), t, "Put should be successful")

	assert.Nil(r.Close(), t)
	_, err = r.Datastore().Get(datastore.NewKey(k))
	assert.Err(err, t, "after closer, Get should be fail")
}
Example #13
0
func (mt *mountBatch) lookupBatch(key ds.Key) (ds.Batch, ds.Key, error) {
	mt.lk.Lock()
	defer mt.lk.Unlock()

	child, loc, rest := mt.d.lookup(key)
	t, ok := mt.mounts[loc.String()]
	if !ok {
		bds, ok := child.(ds.Batching)
		if !ok {
			return nil, ds.NewKey(""), ds.ErrBatchUnsupported
		}
		var err error
		t, err = bds.Batch()
		if err != nil {
			return nil, ds.NewKey(""), err
		}
		mt.mounts[loc.String()] = t
	}
	return t, rest, nil
}
Example #14
0
func Example() {
	mp := ds.NewMapDatastore()
	ns := nsds.Wrap(mp, ds.NewKey("/foo/bar"))

	k := ds.NewKey("/beep")
	v := "boop"

	ns.Put(k, v)
	fmt.Printf("ns.Put %s %s\n", k, v)

	v2, _ := ns.Get(k)
	fmt.Printf("ns.Get %s -> %s\n", k, v2)

	k3 := ds.NewKey("/foo/bar/beep")
	v3, _ := mp.Get(k3)
	fmt.Printf("mp.Get %s -> %s\n", k3, v3)
	// Output:
	// ns.Put /beep boop
	// ns.Get /beep -> boop
	// mp.Get /foo/bar/beep -> boop
}
Example #15
0
func addTestCases(t *testing.T, d Datastore, testcases map[string]string) {
	for k, v := range testcases {
		dsk := ds.NewKey(k)
		if err := d.Put(dsk, []byte(v)); err != nil {
			t.Fatal(err)
		}
	}

	for k, v := range testcases {
		dsk := ds.NewKey(k)
		v2, err := d.Get(dsk)
		if err != nil {
			t.Fatal(err)
		}
		v2b := v2.([]byte)
		if string(v2b) != v {
			t.Errorf("%s values differ: %s != %s", k, v, v2)
		}
	}

}
Example #16
0
func TestDatastorePersistsFromRepoToRepo(t *testing.T) {
	t.Parallel()
	path := testRepoPath("test", t)

	assert.Nil(Init(path, &config.Config{}), t)
	r1, err := Open(path)
	assert.Nil(err, t)

	k := "key"
	expected := []byte(k)
	assert.Nil(r1.Datastore().Put(datastore.NewKey(k), expected), t, "using first repo, Put should be successful")
	assert.Nil(r1.Close(), t)

	r2, err := Open(path)
	assert.Nil(err, t)
	v, err := r2.Datastore().Get(datastore.NewKey(k))
	assert.Nil(err, t, "using second repo, Get should be successful")
	actual, ok := v.([]byte)
	assert.True(ok, t, "value should be the []byte from r1's Put")
	assert.Nil(r2.Close(), t)
	assert.True(bytes.Compare(expected, actual) == 0, t, "data should match")
}
Example #17
0
func RunBatchTest(t *testing.T, ds dstore.BatchingDatastore) {
	batch, err := ds.Batch()
	if err != nil {
		t.Fatal(err)
	}

	r := rand.New()
	var blocks [][]byte
	var keys []dstore.Key
	for i := 0; i < 20; i++ {
		blk := make([]byte, 256*1024)
		r.Read(blk)
		blocks = append(blocks, blk)

		key := dstore.NewKey(base32.StdEncoding.EncodeToString(blk[:8]))
		keys = append(keys, key)

		err := batch.Put(key, blk)
		if err != nil {
			t.Fatal(err)
		}
	}

	// Ensure they are not in the datastore before comitting
	for _, k := range keys {
		_, err := ds.Get(k)
		if err == nil {
			t.Fatal("should not have found this block")
		}
	}

	// commit, write them to the datastore
	err = batch.Commit()
	if err != nil {
		t.Fatal(err)
	}

	for i, k := range keys {
		blk, err := ds.Get(k)
		if err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(blk.([]byte), blocks[i]) {
			t.Fatal("blocks not correct!")
		}
	}
}
Example #18
0
func TestPutGetBytes(t *testing.T) {
	client := clientOrAbort(t)
	ds, err := NewDatastore(client)
	if err != nil {
		t.Fatal(err)
	}
	key, val := datastore.NewKey("foo"), []byte("bar")
	dstest.Nil(ds.Put(key, val), t)
	v, err := ds.Get(key)
	if err != nil {
		t.Fatal(err)
	}
	if bytes.Compare(v.([]byte), val) != 0 {
		t.Fail()
	}
}
Example #19
0
func (d *datastore) runQuery(worker goprocess.Process, qrb *dsq.ResultBuilder) {

	var rnge *util.Range
	if qrb.Query.Prefix != "" {
		rnge = util.BytesPrefix([]byte(qrb.Query.Prefix))
	}
	i := d.DB.NewIterator(rnge, nil)
	defer i.Release()

	// advance iterator for offset
	if qrb.Query.Offset > 0 {
		for j := 0; j < qrb.Query.Offset; j++ {
			i.Next()
		}
	}

	// iterate, and handle limit, too
	for sent := 0; i.Next(); sent++ {
		// end early if we hit the limit
		if qrb.Query.Limit > 0 && sent >= qrb.Query.Limit {
			break
		}

		k := ds.NewKey(string(i.Key())).String()
		e := dsq.Entry{Key: k}

		if !qrb.Query.KeysOnly {
			buf := make([]byte, len(i.Value()))
			copy(buf, i.Value())
			e.Value = buf
		}

		select {
		case qrb.Output <- dsq.Result{Entry: e}: // we sent it out
		case <-worker.Closing(): // client told us to end early.
			break
		}
	}

	if err := i.Error(); err != nil {
		select {
		case qrb.Output <- dsq.Result{Error: err}: // client read our error
		case <-worker.Closing(): // client told us to end.
			return
		}
	}
}
Example #20
0
func TestDelete(t *testing.T) {
	client := clientOrAbort(t)
	ds, err := NewDatastore(client)
	if err != nil {
		t.Fatal(err)
	}
	key, val := datastore.NewKey("foo"), []byte("bar")
	dstest.Nil(ds.Put(key, val), t)
	dstest.Nil(ds.Delete(key), t)

	hasAfterDelete, err := ds.Has(key)
	if err != nil {
		t.Fatal(err)
	}
	if hasAfterDelete {
		t.Fail()
	}
}
Example #21
0
// PrefixTransform constructs a KeyTransform with a pair of functions that
// add or remove the given prefix key.
//
// Warning: will panic if prefix not found when it should be there. This is
// to avoid insidious data inconsistency errors.
func PrefixTransform(prefix ds.Key) ktds.KeyTransform {
	return &ktds.Pair{

		// Convert adds the prefix
		Convert: func(k ds.Key) ds.Key {
			return prefix.Child(k)
		},

		// Invert removes the prefix. panics if prefix not found.
		Invert: func(k ds.Key) ds.Key {
			if !prefix.IsAncestorOf(k) {
				fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k)
				panic("expected prefix not found")
			}

			s := strings.TrimPrefix(k.String(), prefix.String())
			return ds.NewKey(s)
		},
	}
}
Example #22
0
func TestExpiry(t *testing.T) {
	ttl := 1 * time.Second
	client := clientOrAbort(t)
	ds, err := NewExpiringDatastore(client, ttl)
	if err != nil {
		t.Fatal(err)
	}
	key, val := datastore.NewKey("foo"), []byte("bar")
	dstest.Nil(ds.Put(key, val), t)
	time.Sleep(ttl + 1*time.Second)
	dstest.Nil(ds.Delete(key), t)

	hasAfterExpiration, err := ds.Has(key)
	if err != nil {
		t.Fatal(err)
	}
	if hasAfterExpiration {
		t.Fail()
	}
}
Example #23
0
// Query implements Query, inverting keys on the way back out.
func (d *ktds) Query(q dsq.Query) (dsq.Results, error) {
	qr, err := d.child.Query(q)
	if err != nil {
		return nil, err
	}

	ch := make(chan dsq.Result)
	go func() {
		defer close(ch)
		defer qr.Close()

		for r := range qr.Next() {
			if r.Error == nil {
				r.Entry.Key = d.InvertKey(ds.NewKey(r.Entry.Key)).String()
			}
			ch <- r
		}
	}()

	return dsq.DerivedResults(qr, ch), nil
}
Example #24
0
func RunBatchDeleteTest(t *testing.T, ds dstore.BatchingDatastore) {
	r := rand.New()
	var keys []dstore.Key
	for i := 0; i < 20; i++ {
		blk := make([]byte, 16)
		r.Read(blk)

		key := dstore.NewKey(base32.StdEncoding.EncodeToString(blk[:8]))
		keys = append(keys, key)

		err := ds.Put(key, blk)
		if err != nil {
			t.Fatal(err)
		}
	}

	batch, err := ds.Batch()
	if err != nil {
		t.Fatal(err)
	}

	for _, k := range keys {
		err := batch.Delete(k)
		if err != nil {
			t.Fatal(err)
		}
	}
	err = batch.Commit()
	if err != nil {
		t.Fatal(err)
	}

	for _, k := range keys {
		_, err := ds.Get(k)
		if err == nil {
			t.Fatal("shouldnt have found block")
		}
	}
}
Example #25
0
func (d *Datastore) Query(q query.Query) (query.Results, error) {
	if len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("mount only supports listing all prefixed keys in random order")
	}
	key := ds.NewKey(q.Prefix)
	cds, mount, k := d.lookup(key)
	if cds == nil {
		return nil, errors.New("mount only supports listing a mount point")
	}
	// TODO support listing cross mount points too

	// delegate the query to the mounted datastore, while adjusting
	// keys in and out
	q2 := q
	q2.Prefix = k.String()
	wrapDS := keytransform.Wrap(cds, &keytransform.Pair{
		Convert: func(ds.Key) ds.Key {
			panic("this should never be called")
		},
		Invert: func(k ds.Key) ds.Key {
			return mount.Child(k)
		},
	})

	r, err := wrapDS.Query(q2)
	if err != nil {
		return nil, err
	}
	r = query.ResultsReplaceQuery(r, q)
	return r, nil
}
Example #26
0
func TestHasBytes(t *testing.T) {
	client := clientOrAbort(t)
	ds, err := NewDatastore(client)
	if err != nil {
		t.Fatal(err)
	}
	key, val := datastore.NewKey("foo"), []byte("bar")
	has, err := ds.Has(key)
	if err != nil {
		t.Fatal(err)
	}
	if has {
		t.Fail()
	}

	assert.Nil(ds.Put(key, val), t)
	hasAfterPut, err := ds.Has(key)
	if err != nil {
		t.Fatal(err)
	}
	if !hasAfterPut {
		t.Fail()
	}
}
Example #27
0
	"encoding/json"
	"errors"
	"fmt"
	"sync"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	nsds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	key "github.com/ipfs/go-ipfs/blocks/key"
	"github.com/ipfs/go-ipfs/blocks/set"
	mdag "github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/util"
)

var log = util.Logger("pin")
var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys")
var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys")
var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys")

type PinMode int

const (
	Recursive PinMode = iota
	Direct
	Indirect
	NotPinned
)

type Pinner interface {
	IsPinned(key.Key) bool
	Pin(context.Context, *mdag.Node, bool) error
Example #28
0
// AllKeysChan runs a query for keys from the blockstore.
// this is very simplistic, in the future, take dsq.Query as a param?
//
// AllKeysChan respects context
func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {

	// KeysOnly, because that would be _a lot_ of data.
	q := dsq.Query{KeysOnly: true}
	// datastore/namespace does *NOT* fix up Query.Prefix
	q.Prefix = BlockPrefix.String()
	res, err := bs.datastore.Query(q)
	if err != nil {
		return nil, err
	}

	// this function is here to compartmentalize
	get := func() (k key.Key, ok bool) {
		select {
		case <-ctx.Done():
			return k, false
		case e, more := <-res.Next():
			if !more {
				return k, false
			}
			if e.Error != nil {
				log.Debug("blockstore.AllKeysChan got err:", e.Error)
				return k, false
			}

			// need to convert to key.Key using key.KeyFromDsKey.
			k = key.KeyFromDsKey(ds.NewKey(e.Key))
			log.Debug("blockstore: query got key", k)

			// key must be a multihash. else ignore it.
			_, err := mh.Cast([]byte(k))
			if err != nil {
				return "", true
			}

			return k, true
		}
	}

	output := make(chan key.Key)
	go func() {
		defer func() {
			res.Process().Close() // ensure exit (signals early exit, too)
			close(output)
		}()

		for {
			k, ok := get()
			if !ok {
				return
			}
			if k == "" {
				continue
			}

			select {
			case <-ctx.Done():
				return
			case output <- k:
			}
		}
	}()

	return output, nil
}
Example #29
0
	"errors"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
	dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
	mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	blocks "github.com/ipfs/go-ipfs/blocks"
	key "github.com/ipfs/go-ipfs/blocks/key"
	eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
)

var log = eventlog.Logger("blockstore")

// BlockPrefix namespaces blockstore datastores
var BlockPrefix = ds.NewKey("blocks")

var ValueTypeMismatch = errors.New("The retrieved value is not a Block")

var ErrNotFound = errors.New("blockstore: block not found")

// Blockstore wraps a ThreadSafeDatastore
type Blockstore interface {
	DeleteBlock(key.Key) error
	Has(key.Key) (bool, error)
	Get(key.Key) (*blocks.Block, error)
	Put(*blocks.Block) error

	AllKeysChan(ctx context.Context) (<-chan key.Key, error)
}
Example #30
0
func TestCoalesceDelete(t *testing.T) {
	m := setup()
	done := make(chan struct{})
	errs := make(chan error, 30)

	m.ds.Put(ds.NewKey("foo1"), "bar1")
	m.ds.Put(ds.NewKey("foo2"), "bar2")
	m.ds.Put(ds.NewKey("foo3"), "bar3")

	for i := 0; i < 10; i++ {
		go func() {
			err := m.ds.Delete(ds.NewKey("foo1"))
			if err != nil {
				errs <- err
			}
			has, err := m.ds.Has(ds.NewKey("foo1"))
			if err != nil {
				errs <- err
			}
			if has {
				t.Error("still have it after deleting")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			err := m.ds.Delete(ds.NewKey("foo2"))
			if err != nil {
				errs <- err
			}
			has, err := m.ds.Has(ds.NewKey("foo2"))
			if err != nil {
				errs <- err
			}
			if has {
				t.Error("still have it after deleting")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			has, err := m.ds.Has(ds.NewKey("foo3"))
			if err != nil {
				errs <- err
			}
			if !has {
				t.Error("should still have foo3")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			has, err := m.ds.Has(ds.NewKey("foo4"))
			if err != nil {
				errs <- err
			}
			if has {
				t.Error("should not have foo4")
			}
			done <- struct{}{}
		}()
	}

	for i := 0; i < 40; i++ {
		<-done
	}

	if m.inside != 9 {
		t.Error("incalls should be 9", m.inside)
	}

	if m.outside != 63 {
		t.Error("outcalls should be 63", m.outside)
	}
}