func (ks *DSSuite) TestBasic(c *C) { var size = 1000 d, err := lru.NewDatastore(size) c.Check(err, Equals, nil) for i := 0; i < size; i++ { err := d.Put(ds.NewKey(strconv.Itoa(i)), i) c.Check(err, Equals, nil) } for i := 0; i < size; i++ { j, err := d.Get(ds.NewKey(strconv.Itoa(i))) c.Check(j, Equals, i) c.Check(err, Equals, nil) } for i := 0; i < size; i++ { err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i) c.Check(err, Equals, nil) } for i := 0; i < size; i++ { j, err := d.Get(ds.NewKey(strconv.Itoa(i))) c.Check(j, Equals, nil) c.Check(err, Equals, ds.ErrNotFound) } for i := 0; i < size; i++ { j, err := d.Get(ds.NewKey(strconv.Itoa(i + size))) c.Check(j, Equals, i) c.Check(err, Equals, nil) } }
func (ks *DSSuite) TestBasic(c *C) { mpds := ds.NewMapDatastore() nsds := ns.Wrap(mpds, ds.NewKey("abc")) keys := strsToKeys([]string{ "foo", "foo/bar", "foo/bar/baz", "foo/barb", "foo/bar/bazb", "foo/bar/baz/barb", }) for _, k := range keys { err := nsds.Put(k, []byte(k.String())) c.Check(err, Equals, nil) } for _, k := range keys { v1, err := nsds.Get(k) c.Check(err, Equals, nil) c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true) v2, err := mpds.Get(ds.NewKey("abc").Child(k)) c.Check(err, Equals, nil) c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true) } run := func(d ds.Datastore, q dsq.Query) []ds.Key { r, err := d.Query(q) c.Check(err, Equals, nil) e, err := r.Rest() c.Check(err, Equals, nil) return ds.EntryKeys(e) } listA := run(mpds, dsq.Query{}) listB := run(nsds, dsq.Query{}) c.Check(len(listA), Equals, len(listB)) // sort them cause yeah. sort.Sort(ds.KeySlice(listA)) sort.Sort(ds.KeySlice(listB)) for i, kA := range listA { kB := listB[i] c.Check(nsds.InvertKey(kA), Equals, kB) c.Check(kA, Equals, nsds.ConvertKey(kB)) } }
// ConvertKey returns a B58 encoded Datastore key // TODO: this is hacky because it encodes every path component. some // path components may be proper strings already... func (b58KeyConverter) ConvertKey(dsk ds.Key) ds.Key { k := ds.NewKey("/") for _, n := range dsk.Namespaces() { k = k.ChildString(b58.Encode([]byte(n))) } return k }
// Query implements Datastore.Query func (d *Datastore) Query(q query.Query) (query.Results, error) { results := make(chan query.Result) walkFn := func(path string, info os.FileInfo, err error) error { // remove ds path prefix if strings.HasPrefix(path, d.path) { path = path[len(d.path):] } if !info.IsDir() { if strings.HasSuffix(path, ObjectKeySuffix) { path = path[:len(path)-len(ObjectKeySuffix)] } key := ds.NewKey(path) entry := query.Entry{Key: key.String(), Value: query.NotFetched} results <- query.Result{Entry: entry} } return nil } go func() { filepath.Walk(d.path, walkFn) close(results) }() r := query.ResultsWithChan(q, results) r = query.NaiveQueryApply(q, r) return r, nil }
// Query implements Query, inverting keys on the way back out. func (d *datastore) Query(q dsq.Query) (dsq.Results, error) { qr, err := d.raw.Query(q) if err != nil { return nil, err } ch := make(chan dsq.Result) go func() { defer close(ch) defer qr.Close() for r := range qr.Next() { if r.Error != nil { ch <- r continue } k := ds.NewKey(r.Entry.Key) if !d.prefix.IsAncestorOf(k) { continue } r.Entry.Key = d.Datastore.InvertKey(k).String() ch <- r } }() return dsq.DerivedResults(qr, ch), nil }
func strsToKeys(strs []string) []ds.Key { keys := make([]ds.Key, len(strs)) for i, s := range strs { keys[i] = ds.NewKey(s) } return keys }
func Example() { mp := ds.NewMapDatastore() ns := nsds.Wrap(mp, ds.NewKey("/foo/bar")) k := ds.NewKey("/beep") v := "boop" ns.Put(k, v) fmt.Printf("ns.Put %s %s\n", k, v) v2, _ := ns.Get(k) fmt.Printf("ns.Get %s -> %s\n", k, v2) k3 := ds.NewKey("/foo/bar/beep") v3, _ := mp.Get(k3) fmt.Printf("mp.Get %s -> %s\n", k3, v3) // Output: // ns.Put /beep boop // ns.Get /beep -> boop // mp.Get /foo/bar/beep -> boop }
func addTestCases(t *testing.T, d Datastore, testcases map[string]string) { for k, v := range testcases { dsk := ds.NewKey(k) if err := d.Put(dsk, []byte(v)); err != nil { t.Fatal(err) } } for k, v := range testcases { dsk := ds.NewKey(k) v2, err := d.Get(dsk) if err != nil { t.Fatal(err) } v2b := v2.([]byte) if string(v2b) != v { t.Errorf("%s values differ: %s != %s", k, v, v2) } } }
func (d *datastore) runQuery(worker goprocess.Process, qrb *dsq.ResultBuilder) { var rnge *util.Range if qrb.Query.Prefix != "" { rnge = util.BytesPrefix([]byte(qrb.Query.Prefix)) } i := d.DB.NewIterator(rnge, nil) defer i.Release() // advance iterator for offset if qrb.Query.Offset > 0 { for j := 0; j < qrb.Query.Offset; j++ { i.Next() } } // iterate, and handle limit, too for sent := 0; i.Next(); sent++ { // end early if we hit the limit if qrb.Query.Limit > 0 && sent >= qrb.Query.Limit { break } k := ds.NewKey(string(i.Key())).String() e := dsq.Entry{Key: k} if !qrb.Query.KeysOnly { buf := make([]byte, len(i.Value())) copy(buf, i.Value()) e.Value = buf } select { case qrb.Output <- dsq.Result{Entry: e}: // we sent it out case <-worker.Closing(): // client told us to end early. break } } if err := i.Error(); err != nil { select { case qrb.Output <- dsq.Result{Error: err}: // client read our error case <-worker.Closing(): // client told us to end. return } } }
// PrefixTransform constructs a KeyTransform with a pair of functions that // add or remove the given prefix key. // // Warning: will panic if prefix not found when it should be there. This is // to avoid insidious data inconsistency errors. func PrefixTransform(prefix ds.Key) ktds.KeyTransform { return &ktds.Pair{ // Convert adds the prefix Convert: func(k ds.Key) ds.Key { return prefix.Child(k) }, // Invert removes the prefix. panics if prefix not found. Invert: func(k ds.Key) ds.Key { if !prefix.IsAncestorOf(k) { fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k) panic("expected prefix not found") } s := strings.TrimPrefix(k.String(), prefix.String()) return ds.NewKey(s) }, } }
// Query implements Query, inverting keys on the way back out. func (d *ktds) Query(q dsq.Query) (dsq.Results, error) { qr, err := d.child.Query(q) if err != nil { return nil, err } ch := make(chan dsq.Result) go func() { defer close(ch) defer qr.Close() for r := range qr.Next() { if r.Error == nil { r.Entry.Key = d.InvertKey(ds.NewKey(r.Entry.Key)).String() } ch <- r } }() return dsq.DerivedResults(qr, ch), nil }
import ( "errors" blocks "github.com/ipfs/go-blocks" key "github.com/ipfs/go-blocks/key" ds "github.com/ipfs/go-blocks/Godeps/_workspace/src/github.com/jbenet/go-datastore" dsns "github.com/ipfs/go-blocks/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace" dsq "github.com/ipfs/go-blocks/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" mh "github.com/ipfs/go-blocks/Godeps/_workspace/src/github.com/jbenet/go-multihash" context "github.com/ipfs/go-blocks/Godeps/_workspace/src/golang.org/x/net/context" ) // BlockPrefix namespaces blockstore datastores var BlockPrefix = ds.NewKey("blocks") var ValueTypeMismatch = errors.New("The retrieved value is not a Block") var ErrNotFound = errors.New("blockstore: block not found") // Blockstore wraps a ThreadSafeDatastore type Blockstore interface { DeleteBlock(key.Key) error Has(key.Key) (bool, error) Get(key.Key) (*blocks.Block, error) Put(*blocks.Block) error GetChan([]key.Key) <-chan *blocks.Block AllKeysChan(ctx context.Context) (<-chan key.Key, error) }
func TestTiered(t *testing.T) { d1 := ds.NewMapDatastore() d2 := ds.NewMapDatastore() d3 := ds.NewMapDatastore() d4 := ds.NewMapDatastore() td := New(d1, d2, d3, d4) td.Put(ds.NewKey("foo"), "bar") testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar") testHas(t, td.(tiered), ds.NewKey("foo"), "bar") // all children // remove it from, say, caches. d1.Delete(ds.NewKey("foo")) d2.Delete(ds.NewKey("foo")) testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar") testHas(t, td.(tiered)[2:], ds.NewKey("foo"), "bar") testNotHas(t, td.(tiered)[:2], ds.NewKey("foo")) // write it again. td.Put(ds.NewKey("foo"), "bar2") testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar2") testHas(t, td.(tiered), ds.NewKey("foo"), "bar2") }
func (ks *DSSuite) TestBasic(c *C) { pair := &kt.Pair{ Convert: func(k ds.Key) ds.Key { return ds.NewKey("/abc").Child(k) }, Invert: func(k ds.Key) ds.Key { // remove abc prefix l := k.List() if l[0] != "abc" { panic("key does not have prefix. convert failed?") } return ds.KeyWithNamespaces(l[1:]) }, } mpds := ds.NewMapDatastore() ktds := kt.Wrap(mpds, pair) keys := strsToKeys([]string{ "foo", "foo/bar", "foo/bar/baz", "foo/barb", "foo/bar/bazb", "foo/bar/baz/barb", }) for _, k := range keys { err := ktds.Put(k, []byte(k.String())) c.Check(err, Equals, nil) } for _, k := range keys { v1, err := ktds.Get(k) c.Check(err, Equals, nil) c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true) v2, err := mpds.Get(ds.NewKey("abc").Child(k)) c.Check(err, Equals, nil) c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true) } run := func(d ds.Datastore, q dsq.Query) []ds.Key { r, err := d.Query(q) c.Check(err, Equals, nil) e, err := r.Rest() c.Check(err, Equals, nil) return ds.EntryKeys(e) } listA := run(mpds, dsq.Query{}) listB := run(ktds, dsq.Query{}) c.Check(len(listA), Equals, len(listB)) // sort them cause yeah. sort.Sort(ds.KeySlice(listA)) sort.Sort(ds.KeySlice(listB)) for i, kA := range listA { kB := listB[i] c.Check(pair.Invert(kA), Equals, kB) c.Check(kA, Equals, pair.Convert(kB)) } c.Log("listA: ", listA) c.Log("listB: ", listB) }
// DsKey returns a Datastore key func (k Key) DsKey() ds.Key { return ds.NewKey(string(k)) }
func TestTimeCache(t *testing.T) { ttl := time.Millisecond * 100 cache := WithTTL(ttl) cache.Put(ds.NewKey("foo1"), "bar1") cache.Put(ds.NewKey("foo2"), "bar2") <-time.After(ttl / 2) cache.Put(ds.NewKey("foo3"), "bar3") cache.Put(ds.NewKey("foo4"), "bar4") testHas(t, cache, ds.NewKey("foo1"), "bar1") testHas(t, cache, ds.NewKey("foo2"), "bar2") testHas(t, cache, ds.NewKey("foo3"), "bar3") testHas(t, cache, ds.NewKey("foo4"), "bar4") <-time.After(ttl / 2) testNotHas(t, cache, ds.NewKey("foo1")) testNotHas(t, cache, ds.NewKey("foo2")) testHas(t, cache, ds.NewKey("foo3"), "bar3") testHas(t, cache, ds.NewKey("foo4"), "bar4") cache.Delete(ds.NewKey("foo3")) testNotHas(t, cache, ds.NewKey("foo3")) <-time.After(ttl / 2) testNotHas(t, cache, ds.NewKey("foo1")) testNotHas(t, cache, ds.NewKey("foo2")) testNotHas(t, cache, ds.NewKey("foo3")) testNotHas(t, cache, ds.NewKey("foo4")) }
// AllKeysChan runs a query for keys from the blockstore. // this is very simplistic, in the future, take dsq.Query as a param? // // AllKeysChan respects context func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) { // KeysOnly, because that would be _a lot_ of data. q := dsq.Query{KeysOnly: true} // datastore/namespace does *NOT* fix up Query.Prefix q.Prefix = BlockPrefix.String() res, err := bs.datastore.Query(q) if err != nil { return nil, err } // this function is here to compartmentalize get := func() (k key.Key, ok bool) { select { case <-ctx.Done(): return k, false case e, more := <-res.Next(): if !more { return k, false } if e.Error != nil { return k, false } // need to convert to key.Key using key.KeyFromDsKey. k = key.KeyFromDsKey(ds.NewKey(e.Key)) // key must be a multihash. else ignore it. _, err := mh.Cast([]byte(k)) if err != nil { return "", true } return k, true } } output := make(chan key.Key) go func() { defer func() { res.Process().Close() // ensure exit (signals early exit, too) close(output) }() for { k, ok := get() if !ok { return } if k == "" { continue } select { case <-ctx.Done(): return case output <- k: } } }() return output, nil }