func (ks *DSSuite) TestBasic(c *C) {
var size = 1000
d, err := lru.NewDatastore(size)
c.Check(err, Equals, nil)
for i := 0; i < size; i++ {
err := d.Put(ds.NewKey(strconv.Itoa(i)), i)
c.Check(err, Equals, nil)
}
for i := 0; i < size; i++ {
j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
c.Check(j, Equals, i)
c.Check(err, Equals, nil)
}
for i := 0; i < size; i++ {
err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i)
c.Check(err, Equals, nil)
}
for i := 0; i < size; i++ {
j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
c.Check(j, Equals, nil)
c.Check(err, Equals, ds.ErrNotFound)
}
for i := 0; i < size; i++ {
j, err := d.Get(ds.NewKey(strconv.Itoa(i + size)))
c.Check(j, Equals, i)
c.Check(err, Equals, nil)
}
}
func TestCoalesceSamePutDiffPut(t *testing.T) {
m := setup()
done := make(chan struct{})
go func() {
m.ds.Put(ds.NewKey("foo"), "bar")
done <- struct{}{}
}()
go func() {
m.ds.Put(ds.NewKey("foo"), "bar")
done <- struct{}{}
}()
go func() {
m.ds.Put(ds.NewKey("foo"), "bar2")
done <- struct{}{}
}()
go func() {
m.ds.Put(ds.NewKey("foo"), "bar3")
done <- struct{}{}
}()
<-done
<-done
<-done
<-done
if m.inside != 3 {
t.Error("incalls should be 3", m.inside)
}
if m.outside != 4 {
t.Error("outcalls should be 4", m.outside)
}
}
func TestCoalesceHas(t *testing.T) {
m := setup()
done := make(chan struct{})
errs := make(chan error, 30)
m.ds.Put(ds.NewKey("foo1"), "bar")
m.ds.Put(ds.NewKey("foo2"), "baz")
for i := 0; i < 10; i++ {
go func() {
v, err := m.ds.Has(ds.NewKey("foo1"))
if err != nil {
errs <- err
}
if !v {
errs <- fmt.Errorf("should have foo1")
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
v, err := m.ds.Has(ds.NewKey("foo2"))
if err != nil {
errs <- err
}
if !v {
errs <- fmt.Errorf("should have foo2")
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
v, err := m.ds.Has(ds.NewKey("foo3"))
if err != nil {
errs <- err
}
if v {
errs <- fmt.Errorf("should not have foo3")
}
done <- struct{}{}
}()
}
for i := 0; i < 30; i++ {
<-done
}
if m.inside != 5 {
t.Error("incalls should be 3", m.inside)
}
if m.outside != 32 {
t.Error("outcalls should be 30", m.outside)
}
}
func TestCoalesceSameGet(t *testing.T) {
m := setup()
done := make(chan struct{})
errs := make(chan error, 30)
m.ds.Put(ds.NewKey("foo1"), "bar")
m.ds.Put(ds.NewKey("foo2"), "baz")
for i := 0; i < 10; i++ {
go func() {
v, err := m.ds.Get(ds.NewKey("foo1"))
if err != nil {
errs <- err
}
if v != "bar" {
errs <- fmt.Errorf("v is not bar", v)
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
v, err := m.ds.Get(ds.NewKey("foo2"))
if err != nil {
errs <- err
}
if v != "baz" {
errs <- fmt.Errorf("v is not baz", v)
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
_, err := m.ds.Get(ds.NewKey("foo3"))
if err == nil {
errs <- fmt.Errorf("no error")
}
done <- struct{}{}
}()
}
for i := 0; i < 30; i++ {
<-done
}
if m.inside != 5 {
t.Error("incalls should be 3", m.inside)
}
if m.outside != 32 {
t.Error("outcalls should be 30", m.outside)
}
}
func (ks *DSSuite) TestBasic(c *C) {
mpds := ds.NewMapDatastore()
nsds := ns.Wrap(mpds, ds.NewKey("abc"))
keys := strsToKeys([]string{
"foo",
"foo/bar",
"foo/bar/baz",
"foo/barb",
"foo/bar/bazb",
"foo/bar/baz/barb",
})
for _, k := range keys {
err := nsds.Put(k, []byte(k.String()))
c.Check(err, Equals, nil)
}
for _, k := range keys {
v1, err := nsds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
r, err := d.Query(q)
c.Check(err, Equals, nil)
e, err := r.Rest()
c.Check(err, Equals, nil)
return ds.EntryKeys(e)
}
listA := run(mpds, dsq.Query{})
listB := run(nsds, dsq.Query{})
c.Check(len(listA), Equals, len(listB))
// sort them cause yeah.
sort.Sort(ds.KeySlice(listA))
sort.Sort(ds.KeySlice(listB))
for i, kA := range listA {
kB := listB[i]
c.Check(nsds.InvertKey(kA), Equals, kB)
c.Check(kA, Equals, nsds.ConvertKey(kB))
}
}
// Query implements Datastore.Query
func (d *Datastore) Query(q query.Query) (query.Results, error) {
results := make(chan query.Result)
walkFn := func(path string, info os.FileInfo, err error) error {
// remove ds path prefix
if strings.HasPrefix(path, d.path) {
path = path[len(d.path):]
}
if !info.IsDir() {
if strings.HasSuffix(path, ObjectKeySuffix) {
path = path[:len(path)-len(ObjectKeySuffix)]
}
key := ds.NewKey(path)
entry := query.Entry{Key: key.String(), Value: query.NotFetched}
results <- query.Result{Entry: entry}
}
return nil
}
go func() {
filepath.Walk(d.path, walkFn)
close(results)
}()
r := query.ResultsWithChan(q, results)
r = query.NaiveQueryApply(q, r)
return r, nil
}
func strsToKeys(strs []string) []ds.Key {
keys := make([]ds.Key, len(strs))
for i, s := range strs {
keys[i] = ds.NewKey(s)
}
return keys
}
// ConvertKey returns a B58 encoded Datastore key
// TODO: this is hacky because it encodes every path component. some
// path components may be proper strings already...
func (b58KeyConverter) ConvertKey(dsk ds.Key) ds.Key {
k := ds.NewKey("/")
for _, n := range dsk.Namespaces() {
k = k.ChildString(b58.Encode([]byte(n)))
}
return k
}
// Query implements Query, inverting keys on the way back out.
func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
qr, err := d.raw.Query(q)
if err != nil {
return nil, err
}
ch := make(chan dsq.Result)
go func() {
defer close(ch)
defer qr.Close()
for r := range qr.Next() {
if r.Error != nil {
ch <- r
continue
}
k := ds.NewKey(r.Entry.Key)
if !d.prefix.IsAncestorOf(k) {
continue
}
r.Entry.Key = d.Datastore.InvertKey(k).String()
ch <- r
}
}()
return dsq.DerivedResults(qr, ch), nil
}
func Example() {
mp := ds.NewMapDatastore()
ns := nsds.Wrap(mp, ds.NewKey("/foo/bar"))
k := ds.NewKey("/beep")
v := "boop"
ns.Put(k, v)
fmt.Printf("ns.Put %s %s\n", k, v)
v2, _ := ns.Get(k)
fmt.Printf("ns.Get %s -> %s\n", k, v2)
k3 := ds.NewKey("/foo/bar/beep")
v3, _ := mp.Get(k3)
fmt.Printf("mp.Get %s -> %s\n", k3, v3)
// Output:
// ns.Put /beep boop
// ns.Get /beep -> boop
// mp.Get /foo/bar/beep -> boop
}
func addTestCases(t *testing.T, d Datastore, testcases map[string]string) {
for k, v := range testcases {
dsk := ds.NewKey(k)
if err := d.Put(dsk, []byte(v)); err != nil {
t.Fatal(err)
}
}
for k, v := range testcases {
dsk := ds.NewKey(k)
v2, err := d.Get(dsk)
if err != nil {
t.Fatal(err)
}
v2b := v2.([]byte)
if string(v2b) != v {
t.Errorf("%s values differ: %s != %s", k, v, v2)
}
}
}
func RunBatchTest(t *testing.T, ds dstore.BatchingDatastore) {
batch, err := ds.Batch()
if err != nil {
t.Fatal(err)
}
r := rand.New()
var blocks [][]byte
var keys []dstore.Key
for i := 0; i < 20; i++ {
blk := make([]byte, 256*1024)
r.Read(blk)
blocks = append(blocks, blk)
key := dstore.NewKey(base32.StdEncoding.EncodeToString(blk[:8]))
keys = append(keys, key)
err := batch.Put(key, blk)
if err != nil {
t.Fatal(err)
}
}
// Ensure they are not in the datastore before comitting
for _, k := range keys {
_, err := ds.Get(k)
if err == nil {
t.Fatal("should not have found this block")
}
}
// commit, write them to the datastore
err = batch.Commit()
if err != nil {
t.Fatal(err)
}
for i, k := range keys {
blk, err := ds.Get(k)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(blk.([]byte), blocks[i]) {
t.Fatal("blocks not correct!")
}
}
}
func TestPutGetBytes(t *testing.T) {
client := clientOrAbort(t)
ds, err := NewDatastore(client)
if err != nil {
t.Fatal(err)
}
key, val := datastore.NewKey("foo"), []byte("bar")
dstest.Nil(ds.Put(key, val), t)
v, err := ds.Get(key)
if err != nil {
t.Fatal(err)
}
if bytes.Compare(v.([]byte), val) != 0 {
t.Fail()
}
}
func TestDelete(t *testing.T) {
client := clientOrAbort(t)
ds, err := NewDatastore(client)
if err != nil {
t.Fatal(err)
}
key, val := datastore.NewKey("foo"), []byte("bar")
dstest.Nil(ds.Put(key, val), t)
dstest.Nil(ds.Delete(key), t)
hasAfterDelete, err := ds.Has(key)
if err != nil {
t.Fatal(err)
}
if hasAfterDelete {
t.Fail()
}
}
// PrefixTransform constructs a KeyTransform with a pair of functions that
// add or remove the given prefix key.
//
// Warning: will panic if prefix not found when it should be there. This is
// to avoid insidious data inconsistency errors.
func PrefixTransform(prefix ds.Key) ktds.KeyTransform {
return &ktds.Pair{
// Convert adds the prefix
Convert: func(k ds.Key) ds.Key {
return prefix.Child(k)
},
// Invert removes the prefix. panics if prefix not found.
Invert: func(k ds.Key) ds.Key {
if !prefix.IsAncestorOf(k) {
fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k)
panic("expected prefix not found")
}
s := strings.TrimPrefix(k.String(), prefix.String())
return ds.NewKey(s)
},
}
}
func TestExpiry(t *testing.T) {
ttl := 1 * time.Second
client := clientOrAbort(t)
ds, err := NewExpiringDatastore(client, ttl)
if err != nil {
t.Fatal(err)
}
key, val := datastore.NewKey("foo"), []byte("bar")
dstest.Nil(ds.Put(key, val), t)
time.Sleep(ttl + 1*time.Second)
dstest.Nil(ds.Delete(key), t)
hasAfterExpiration, err := ds.Has(key)
if err != nil {
t.Fatal(err)
}
if hasAfterExpiration {
t.Fail()
}
}
// Query implements Query, inverting keys on the way back out.
func (d *ktds) Query(q dsq.Query) (dsq.Results, error) {
qr, err := d.child.Query(q)
if err != nil {
return nil, err
}
ch := make(chan dsq.Result)
go func() {
defer close(ch)
defer qr.Close()
for r := range qr.Next() {
if r.Error == nil {
r.Entry.Key = d.InvertKey(ds.NewKey(r.Entry.Key)).String()
}
ch <- r
}
}()
return dsq.DerivedResults(qr, ch), nil
}
func RunBatchDeleteTest(t *testing.T, ds dstore.BatchingDatastore) {
r := rand.New()
var keys []dstore.Key
for i := 0; i < 20; i++ {
blk := make([]byte, 16)
r.Read(blk)
key := dstore.NewKey(base32.StdEncoding.EncodeToString(blk[:8]))
keys = append(keys, key)
err := ds.Put(key, blk)
if err != nil {
t.Fatal(err)
}
}
batch, err := ds.Batch()
if err != nil {
t.Fatal(err)
}
for _, k := range keys {
err := batch.Delete(k)
if err != nil {
t.Fatal(err)
}
}
err = batch.Commit()
if err != nil {
t.Fatal(err)
}
for _, k := range keys {
_, err := ds.Get(k)
if err == nil {
t.Fatal("shouldnt have found block")
}
}
}
"errors"
blocks "github.com/heems/bssim/Godeps/_workspace/src/github.com/ipfs/go-ipfs/blocks"
key "github.com/heems/bssim/Godeps/_workspace/src/github.com/ipfs/go-ipfs/blocks/key"
eventlog "github.com/heems/bssim/Godeps/_workspace/src/github.com/ipfs/go-ipfs/thirdparty/eventlog"
ds "github.com/heems/bssim/Godeps/_workspace/src/github.com/jbenet/go-datastore"
dsns "github.com/heems/bssim/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
dsq "github.com/heems/bssim/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
mh "github.com/heems/bssim/Godeps/_workspace/src/github.com/jbenet/go-multihash"
context "github.com/heems/bssim/Godeps/_workspace/src/golang.org/x/net/context"
)
var log = eventlog.Logger("blockstore")
// BlockPrefix namespaces blockstore datastores
var BlockPrefix = ds.NewKey("blocks")
var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
var ErrNotFound = errors.New("blockstore: block not found")
// Blockstore wraps a ThreadSafeDatastore
type Blockstore interface {
DeleteBlock(key.Key) error
Has(key.Key) (bool, error)
Get(key.Key) (*blocks.Block, error)
Put(*blocks.Block) error
PutMany([]*blocks.Block) error
AllKeysChan(ctx context.Context) (<-chan key.Key, error)
}
func (ps *peerstore) Get(p ID, key string) (interface{}, error) {
dsk := ds.NewKey(string(p) + "/" + key)
return ps.ds.Get(dsk)
}
func (ps *peerstore) Put(p ID, key string, val interface{}) error {
dsk := ds.NewKey(string(p) + "/" + key)
return ps.ds.Put(dsk, val)
}
// DsKey returns a Datastore key
func (k Key) DsKey() ds.Key {
return ds.NewKey(string(k))
}
func TestTiered(t *testing.T) {
d1 := ds.NewMapDatastore()
d2 := ds.NewMapDatastore()
d3 := ds.NewMapDatastore()
d4 := ds.NewMapDatastore()
td := New(d1, d2, d3, d4)
td.Put(ds.NewKey("foo"), "bar")
testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
testHas(t, td.(tiered), ds.NewKey("foo"), "bar") // all children
// remove it from, say, caches.
d1.Delete(ds.NewKey("foo"))
d2.Delete(ds.NewKey("foo"))
testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
testHas(t, td.(tiered)[2:], ds.NewKey("foo"), "bar")
testNotHas(t, td.(tiered)[:2], ds.NewKey("foo"))
// write it again.
td.Put(ds.NewKey("foo"), "bar2")
testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar2")
testHas(t, td.(tiered), ds.NewKey("foo"), "bar2")
}
func (ks *DSSuite) TestBasic(c *C) {
pair := &kt.Pair{
Convert: func(k ds.Key) ds.Key {
return ds.NewKey("/abc").Child(k)
},
Invert: func(k ds.Key) ds.Key {
// remove abc prefix
l := k.List()
if l[0] != "abc" {
panic("key does not have prefix. convert failed?")
}
return ds.KeyWithNamespaces(l[1:])
},
}
mpds := ds.NewMapDatastore()
ktds := kt.Wrap(mpds, pair)
keys := strsToKeys([]string{
"foo",
"foo/bar",
"foo/bar/baz",
"foo/barb",
"foo/bar/bazb",
"foo/bar/baz/barb",
})
for _, k := range keys {
err := ktds.Put(k, []byte(k.String()))
c.Check(err, Equals, nil)
}
for _, k := range keys {
v1, err := ktds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
r, err := d.Query(q)
c.Check(err, Equals, nil)
e, err := r.Rest()
c.Check(err, Equals, nil)
return ds.EntryKeys(e)
}
listA := run(mpds, dsq.Query{})
listB := run(ktds, dsq.Query{})
c.Check(len(listA), Equals, len(listB))
// sort them cause yeah.
sort.Sort(ds.KeySlice(listA))
sort.Sort(ds.KeySlice(listB))
for i, kA := range listA {
kB := listB[i]
c.Check(pair.Invert(kA), Equals, kB)
c.Check(kA, Equals, pair.Convert(kB))
}
c.Log("listA: ", listA)
c.Log("listB: ", listB)
}
// AllKeysChan runs a query for keys from the blockstore.
// this is very simplistic, in the future, take dsq.Query as a param?
//
// AllKeysChan respects context
func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
// KeysOnly, because that would be _a lot_ of data.
q := dsq.Query{KeysOnly: true}
// datastore/namespace does *NOT* fix up Query.Prefix
q.Prefix = BlockPrefix.String()
res, err := bs.datastore.Query(q)
if err != nil {
return nil, err
}
// this function is here to compartmentalize
get := func() (k key.Key, ok bool) {
select {
case <-ctx.Done():
return k, false
case e, more := <-res.Next():
if !more {
return k, false
}
if e.Error != nil {
log.Debug("blockstore.AllKeysChan got err:", e.Error)
return k, false
}
// need to convert to key.Key using key.KeyFromDsKey.
k = key.KeyFromDsKey(ds.NewKey(e.Key))
log.Debug("blockstore: query got key", k)
// key must be a multihash. else ignore it.
_, err := mh.Cast([]byte(k))
if err != nil {
return "", true
}
return k, true
}
}
output := make(chan key.Key)
go func() {
defer func() {
res.Process().Close() // ensure exit (signals early exit, too)
close(output)
}()
for {
k, ok := get()
if !ok {
return
}
if k == "" {
continue
}
select {
case <-ctx.Done():
return
case output <- k:
}
}
}()
return output, nil
}
func TestTimeCache(t *testing.T) {
ttl := time.Millisecond * 100
cache := WithTTL(ttl)
cache.Put(ds.NewKey("foo1"), "bar1")
cache.Put(ds.NewKey("foo2"), "bar2")
<-time.After(ttl / 2)
cache.Put(ds.NewKey("foo3"), "bar3")
cache.Put(ds.NewKey("foo4"), "bar4")
testHas(t, cache, ds.NewKey("foo1"), "bar1")
testHas(t, cache, ds.NewKey("foo2"), "bar2")
testHas(t, cache, ds.NewKey("foo3"), "bar3")
testHas(t, cache, ds.NewKey("foo4"), "bar4")
<-time.After(ttl / 2)
testNotHas(t, cache, ds.NewKey("foo1"))
testNotHas(t, cache, ds.NewKey("foo2"))
testHas(t, cache, ds.NewKey("foo3"), "bar3")
testHas(t, cache, ds.NewKey("foo4"), "bar4")
cache.Delete(ds.NewKey("foo3"))
testNotHas(t, cache, ds.NewKey("foo3"))
<-time.After(ttl / 2)
testNotHas(t, cache, ds.NewKey("foo1"))
testNotHas(t, cache, ds.NewKey("foo2"))
testNotHas(t, cache, ds.NewKey("foo3"))
testNotHas(t, cache, ds.NewKey("foo4"))
}
func TestCoalesceDelete(t *testing.T) {
m := setup()
done := make(chan struct{})
errs := make(chan error, 30)
m.ds.Put(ds.NewKey("foo1"), "bar1")
m.ds.Put(ds.NewKey("foo2"), "bar2")
m.ds.Put(ds.NewKey("foo3"), "bar3")
for i := 0; i < 10; i++ {
go func() {
err := m.ds.Delete(ds.NewKey("foo1"))
if err != nil {
errs <- err
}
has, err := m.ds.Has(ds.NewKey("foo1"))
if err != nil {
errs <- err
}
if has {
t.Error("still have it after deleting")
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
err := m.ds.Delete(ds.NewKey("foo2"))
if err != nil {
errs <- err
}
has, err := m.ds.Has(ds.NewKey("foo2"))
if err != nil {
errs <- err
}
if has {
t.Error("still have it after deleting")
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
has, err := m.ds.Has(ds.NewKey("foo3"))
if err != nil {
errs <- err
}
if !has {
t.Error("should still have foo3")
}
done <- struct{}{}
}()
}
for i := 0; i < 10; i++ {
go func() {
has, err := m.ds.Has(ds.NewKey("foo4"))
if err != nil {
errs <- err
}
if has {
t.Error("should not have foo4")
}
done <- struct{}{}
}()
}
for i := 0; i < 40; i++ {
<-done
}
if m.inside != 9 {
t.Error("incalls should be 9", m.inside)
}
if m.outside != 63 {
t.Error("outcalls should be 63", m.outside)
}
}
