func TestQueryCallsLast(t *testing.T) {
var d1n, d2n, d3n int
d1 := dscb.Wrap(ds.NewMapDatastore(), func() { d1n++ })
d2 := dscb.Wrap(ds.NewMapDatastore(), func() { d2n++ })
d3 := dscb.Wrap(ds.NewMapDatastore(), func() { d3n++ })
td := New(d1, d2, d3)
td.Query(dsq.Query{})
if d3n < 1 {
t.Error("should call last")
}
}
func TestPutThenGetBlocks(t *testing.T) {
checkErr := func(err error) {
if err != nil {
t.Fatal(err)
}
}
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
block1 := blocks.NewBlock([]byte("some data 1"))
block2 := blocks.NewBlock([]byte("some data 2"))
block3 := blocks.NewBlock([]byte("some data 3"))
block4 := blocks.NewBlock([]byte("some data 4"))
checkErr(bs.Put(block1))
checkErr(bs.Put(block2))
checkErr(bs.Put(block3))
out := bs.GetChan([]key.Key{block1.Key(), block2.Key(), block3.Key(), block4.Key()})
checkGets := func(expect *blocks.Block, out <-chan *blocks.Block) {
actual := <-out
if !bytes.Equal(expect.Data, actual.Data) {
t.Fatal("blocks differ")
}
}
checkGets(block1, out)
checkGets(block2, out)
checkGets(block3, out)
_, more := <-out
if more {
t.Fatal("should be closed")
}
}
func TestReturnsErrorWhenSizeNegative(t *testing.T) {
bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
_, err := WriteCached(bs, -1)
if err != nil {
return
}
t.Fail()
}
func TestGetWhenKeyNotPresent(t *testing.T) {
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
_, err := bs.Get(key.Key("not present"))
if err != nil {
t.Log("As expected, block is not present")
return
}
t.Fail()
}
func (ks *DSSuite) TestBasic(c *C) {
mpds := ds.NewMapDatastore()
nsds := ns.Wrap(mpds, ds.NewKey("abc"))
keys := strsToKeys([]string{
"foo",
"foo/bar",
"foo/bar/baz",
"foo/barb",
"foo/bar/bazb",
"foo/bar/baz/barb",
})
for _, k := range keys {
err := nsds.Put(k, []byte(k.String()))
c.Check(err, Equals, nil)
}
for _, k := range keys {
v1, err := nsds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
r, err := d.Query(q)
c.Check(err, Equals, nil)
e, err := r.Rest()
c.Check(err, Equals, nil)
return ds.EntryKeys(e)
}
listA := run(mpds, dsq.Query{})
listB := run(nsds, dsq.Query{})
c.Check(len(listA), Equals, len(listB))
// sort them cause yeah.
sort.Sort(ds.KeySlice(listA))
sort.Sort(ds.KeySlice(listB))
for i, kA := range listA {
kB := listB[i]
c.Check(nsds.InvertKey(kA), Equals, kB)
c.Check(kA, Equals, nsds.ConvertKey(kB))
}
}
func TestValueTypeMismatch(t *testing.T) {
block := blocks.NewBlock([]byte("some data"))
datastore := ds.NewMapDatastore()
k := BlockPrefix.Child(block.Key().DsKey())
datastore.Put(k, "data that isn't a block!")
blockstore := NewBlockstore(ds_sync.MutexWrap(datastore))
_, err := blockstore.Get(block.Key())
if err != ValueTypeMismatch {
t.Fatal(err)
}
}
func TestTiered(t *testing.T) {
d1 := ds.NewMapDatastore()
d2 := ds.NewMapDatastore()
d3 := ds.NewMapDatastore()
d4 := ds.NewMapDatastore()
td := New(d1, d2, d3, d4)
td.Put(ds.NewKey("foo"), "bar")
testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
testHas(t, td.(tiered), ds.NewKey("foo"), "bar") // all children
// remove it from, say, caches.
d1.Delete(ds.NewKey("foo"))
d2.Delete(ds.NewKey("foo"))
testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
testHas(t, td.(tiered)[2:], ds.NewKey("foo"), "bar")
testNotHas(t, td.(tiered)[:2], ds.NewKey("foo"))
// write it again.
td.Put(ds.NewKey("foo"), "bar2")
testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar2")
testHas(t, td.(tiered), ds.NewKey("foo"), "bar2")
}
func TestElideDuplicateWrite(t *testing.T) {
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
bs := NewBlockstore(syncds.MutexWrap(cd))
cachedbs, err := WriteCached(bs, 1)
if err != nil {
t.Fatal(err)
}
b1 := blocks.NewBlock([]byte("foo"))
cachedbs.Put(b1)
cd.SetFunc(func() {
t.Fatal("write hit the datastore")
})
cachedbs.Put(b1)
}
func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []key.Key) {
if d == nil {
d = ds.NewMapDatastore()
}
bs := NewBlockstore(ds_sync.MutexWrap(d))
keys := make([]key.Key, N)
for i := 0; i < N; i++ {
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := bs.Put(block)
if err != nil {
t.Fatal(err)
}
keys[i] = block.Key()
}
return bs, keys
}
func TestPutThenGetBlock(t *testing.T) {
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
block := blocks.NewBlock([]byte("some data"))
err := bs.Put(block)
if err != nil {
t.Fatal(err)
}
blockFromBlockstore, err := bs.Get(block.Key())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(block.Data, blockFromBlockstore.Data) {
t.Fatal("blocks differ")
}
}
func TestRemoveCacheEntryOnDelete(t *testing.T) {
b := blocks.NewBlock([]byte("foo"))
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
bs := NewBlockstore(syncds.MutexWrap(cd))
cachedbs, err := WriteCached(bs, 1)
if err != nil {
t.Fatal(err)
}
cachedbs.Put(b)
writeHitTheDatastore := false
cd.SetFunc(func() {
writeHitTheDatastore = true
})
cachedbs.DeleteBlock(b.Key())
cachedbs.Put(b)
if !writeHitTheDatastore {
t.Fail()
}
}
func Example() {
mp := ds.NewMapDatastore()
ns := nsds.Wrap(mp, ds.NewKey("/foo/bar"))
k := ds.NewKey("/beep")
v := "boop"
ns.Put(k, v)
fmt.Printf("ns.Put %s %s\n", k, v)
v2, _ := ns.Get(k)
fmt.Printf("ns.Get %s -> %s\n", k, v2)
k3 := ds.NewKey("/foo/bar/beep")
v3, _ := mp.Get(k3)
fmt.Printf("mp.Get %s -> %s\n", k3, v3)
// Output:
// ns.Put /beep boop
// ns.Get /beep -> boop
// mp.Get /foo/bar/beep -> boop
}
func TestAllKeysRespectsContext(t *testing.T) {
N := 100
d := &queryTestDS{ds: ds.NewMapDatastore()}
bs, _ := newBlockStoreWithKeys(t, d, N)
started := make(chan struct{}, 1)
done := make(chan struct{}, 1)
errors := make(chan error, 100)
getKeys := func(ctx context.Context) {
started <- struct{}{}
ch, err := bs.AllKeysChan(ctx) // once without cancelling
if err != nil {
errors <- err
}
_ = collect(ch)
done <- struct{}{}
errors <- nil // a nil one to signal break
}
// Once without context, to make sure it all works
{
var results dsq.Results
var resultsmu = make(chan struct{})
resultChan := make(chan dsq.Result)
d.SetFunc(func(q dsq.Query) (dsq.Results, error) {
results = dsq.ResultsWithChan(q, resultChan)
resultsmu <- struct{}{}
return results, nil
})
go getKeys(context.Background())
// make sure it's waiting.
<-started
<-resultsmu
select {
case <-done:
t.Fatal("sync is wrong")
case <-results.Process().Closing():
t.Fatal("should not be closing")
case <-results.Process().Closed():
t.Fatal("should not be closed")
default:
}
e := dsq.Entry{Key: BlockPrefix.ChildString("foo").String()}
resultChan <- dsq.Result{Entry: e} // let it go.
close(resultChan)
<-done // should be done now.
<-results.Process().Closed() // should be closed now
// print any errors
for err := range errors {
if err == nil {
break
}
t.Error(err)
}
}
// Once with
{
var results dsq.Results
var resultsmu = make(chan struct{})
resultChan := make(chan dsq.Result)
d.SetFunc(func(q dsq.Query) (dsq.Results, error) {
results = dsq.ResultsWithChan(q, resultChan)
resultsmu <- struct{}{}
return results, nil
})
ctx, cancel := context.WithCancel(context.Background())
go getKeys(ctx)
// make sure it's waiting.
<-started
<-resultsmu
select {
case <-done:
t.Fatal("sync is wrong")
case <-results.Process().Closing():
t.Fatal("should not be closing")
case <-results.Process().Closed():
t.Fatal("should not be closed")
default:
}
cancel() // let it go.
select {
case <-done:
t.Fatal("sync is wrong")
case <-results.Process().Closed():
t.Fatal("should not be closed") // should not be closed yet.
case <-results.Process().Closing():
// should be closing now!
t.Log("closing correctly at this point.")
}
close(resultChan)
<-done // should be done now.
<-results.Process().Closed() // should be closed now
// print any errors
for err := range errors {
if err == nil {
break
}
t.Error(err)
}
}
}
func (ks *DSSuite) TestBasic(c *C) {
pair := &kt.Pair{
Convert: func(k ds.Key) ds.Key {
return ds.NewKey("/abc").Child(k)
},
Invert: func(k ds.Key) ds.Key {
// remove abc prefix
l := k.List()
if l[0] != "abc" {
panic("key does not have prefix. convert failed?")
}
return ds.KeyWithNamespaces(l[1:])
},
}
mpds := ds.NewMapDatastore()
ktds := kt.Wrap(mpds, pair)
keys := strsToKeys([]string{
"foo",
"foo/bar",
"foo/bar/baz",
"foo/barb",
"foo/bar/bazb",
"foo/bar/baz/barb",
})
for _, k := range keys {
err := ktds.Put(k, []byte(k.String()))
c.Check(err, Equals, nil)
}
for _, k := range keys {
v1, err := ktds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
r, err := d.Query(q)
c.Check(err, Equals, nil)
e, err := r.Rest()
c.Check(err, Equals, nil)
return ds.EntryKeys(e)
}
listA := run(mpds, dsq.Query{})
listB := run(ktds, dsq.Query{})
c.Check(len(listA), Equals, len(listB))
// sort them cause yeah.
sort.Sort(ds.KeySlice(listA))
sort.Sort(ds.KeySlice(listB))
for i, kA := range listA {
kB := listB[i]
c.Check(pair.Invert(kA), Equals, kB)
c.Check(kA, Equals, pair.Convert(kB))
}
c.Log("listA: ", listA)
c.Log("listB: ", listB)
}
func WithTTL(ttl time.Duration) ds.Datastore {
return WithCache(ds.NewMapDatastore(), ttl)
}
func bstore() blockstore.Blockstore {
return blockstore.NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
}