func TestCoalesceSamePutDiffPut(t *testing.T) { m := setup() done := make(chan struct{}) go func() { m.ds.Put(ds.NewKey("foo"), "bar") done <- struct{}{} }() go func() { m.ds.Put(ds.NewKey("foo"), "bar") done <- struct{}{} }() go func() { m.ds.Put(ds.NewKey("foo"), "bar2") done <- struct{}{} }() go func() { m.ds.Put(ds.NewKey("foo"), "bar3") done <- struct{}{} }() <-done <-done <-done <-done if m.inside != 3 { t.Error("incalls should be 3", m.inside) } if m.outside != 4 { t.Error("outcalls should be 4", m.outside) } }
func (ks *DSSuite) TestBasic(c *C) { var size = 1000 d, err := lru.NewDatastore(size) c.Check(err, Equals, nil) for i := 0; i < size; i++ { err := d.Put(ds.NewKey(strconv.Itoa(i)), i) c.Check(err, Equals, nil) } for i := 0; i < size; i++ { j, err := d.Get(ds.NewKey(strconv.Itoa(i))) c.Check(j, Equals, i) c.Check(err, Equals, nil) } for i := 0; i < size; i++ { err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i) c.Check(err, Equals, nil) } for i := 0; i < size; i++ { j, err := d.Get(ds.NewKey(strconv.Itoa(i))) c.Check(j, Equals, nil) c.Check(err, Equals, ds.ErrNotFound) } for i := 0; i < size; i++ { j, err := d.Get(ds.NewKey(strconv.Itoa(i + size))) c.Check(j, Equals, i) c.Check(err, Equals, nil) } }
func TestCoalesceHas(t *testing.T) { m := setup() done := make(chan struct{}) errs := make(chan error, 30) m.ds.Put(ds.NewKey("foo1"), "bar") m.ds.Put(ds.NewKey("foo2"), "baz") for i := 0; i < 10; i++ { go func() { v, err := m.ds.Has(ds.NewKey("foo1")) if err != nil { errs <- err } if !v { errs <- fmt.Errorf("should have foo1") } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { v, err := m.ds.Has(ds.NewKey("foo2")) if err != nil { errs <- err } if !v { errs <- fmt.Errorf("should have foo2") } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { v, err := m.ds.Has(ds.NewKey("foo3")) if err != nil { errs <- err } if v { errs <- fmt.Errorf("should not have foo3") } done <- struct{}{} }() } for i := 0; i < 30; i++ { <-done } if m.inside != 5 { t.Error("incalls should be 3", m.inside) } if m.outside != 32 { t.Error("outcalls should be 30", m.outside) } }
// openDatastore returns an error if the config file is not present. func (r *FSRepo) openDatastore() error { leveldbPath := path.Join(r.path, leveldbDirectory) var err error // save leveldb reference so it can be neatly closed afterward r.leveldbDS, err = levelds.NewDatastore(leveldbPath, &levelds.Options{ Compression: ldbopts.NoCompression, }) if err != nil { return errors.New("unable to open leveldb datastore") } // 4TB of 256kB objects ~=17M objects, splitting that 256-way // leads to ~66k objects per dir, splitting 256*256-way leads to // only 256. // // The keys seen by the block store have predictable prefixes, // including "/" from datastore.Key and 2 bytes from multihash. To // reach a uniform 256-way split, we need approximately 4 bytes of // prefix. blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4) if err != nil { return errors.New("unable to open flatfs datastore") } // Add our PeerID to metrics paths to keep them unique // // As some tests just pass a zero-value Config to fsrepo.Init, // cope with missing PeerID. id := r.config.Identity.PeerID if id == "" { // the tests pass in a zero Config; cope with it id = fmt.Sprintf("uninitialized_%p", r) } prefix := "fsrepo." + id + ".datastore." r.metricsBlocks = measure.New(prefix+"blocks", blocksDS) r.metricsLevelDB = measure.New(prefix+"leveldb", r.leveldbDS) mountDS := mount.New([]mount.Mount{ { Prefix: ds.NewKey("/blocks"), Datastore: r.metricsBlocks, }, { Prefix: ds.NewKey("/"), Datastore: r.metricsLevelDB, }, }) // Make sure it's ok to claim the virtual datastore from mount as // threadsafe. There's no clean way to make mount itself provide // this information without copy-pasting the code into two // variants. This is the same dilemma as the `[].byte` attempt at // introducing const types to Go. var _ ds.ThreadSafeDatastore = blocksDS var _ ds.ThreadSafeDatastore = r.leveldbDS r.ds = ds2.ClaimThreadSafe{mountDS} return nil }
func TestCoalesceSameGet(t *testing.T) { m := setup() done := make(chan struct{}) errs := make(chan error, 30) m.ds.Put(ds.NewKey("foo1"), "bar") m.ds.Put(ds.NewKey("foo2"), "baz") for i := 0; i < 10; i++ { go func() { v, err := m.ds.Get(ds.NewKey("foo1")) if err != nil { errs <- err } if v != "bar" { errs <- fmt.Errorf("v is not bar", v) } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { v, err := m.ds.Get(ds.NewKey("foo2")) if err != nil { errs <- err } if v != "baz" { errs <- fmt.Errorf("v is not baz", v) } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { _, err := m.ds.Get(ds.NewKey("foo3")) if err == nil { errs <- fmt.Errorf("no error") } done <- struct{}{} }() } for i := 0; i < 30; i++ { <-done } if m.inside != 5 { t.Error("incalls should be 3", m.inside) } if m.outside != 32 { t.Error("outcalls should be 30", m.outside) } }
func (d *Datastore) lookup(key ds.Key) (ds.Datastore, ds.Key, ds.Key) { d.lk.Lock() defer d.lk.Unlock() for _, m := range d.mounts { if m.Prefix.Equal(key) || m.Prefix.IsAncestorOf(key) { s := strings.TrimPrefix(key.String(), m.Prefix.String()) k := ds.NewKey(s) return m.Datastore, m.Prefix, k } } return nil, ds.NewKey("/"), key }
func (ks *DSSuite) TestBasic(c *C) { mpds := ds.NewMapDatastore() nsds := ns.Wrap(mpds, ds.NewKey("abc")) keys := strsToKeys([]string{ "foo", "foo/bar", "foo/bar/baz", "foo/barb", "foo/bar/bazb", "foo/bar/baz/barb", }) for _, k := range keys { err := nsds.Put(k, []byte(k.String())) c.Check(err, Equals, nil) } for _, k := range keys { v1, err := nsds.Get(k) c.Check(err, Equals, nil) c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true) v2, err := mpds.Get(ds.NewKey("abc").Child(k)) c.Check(err, Equals, nil) c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true) } run := func(d ds.Datastore, q dsq.Query) []ds.Key { r, err := d.Query(q) c.Check(err, Equals, nil) e, err := r.Rest() c.Check(err, Equals, nil) return ds.EntryKeys(e) } listA := run(mpds, dsq.Query{}) listB := run(nsds, dsq.Query{}) c.Check(len(listA), Equals, len(listB)) // sort them cause yeah. sort.Sort(ds.KeySlice(listA)) sort.Sort(ds.KeySlice(listB)) for i, kA := range listA { kB := listB[i] c.Check(nsds.InvertKey(kA), Equals, kB) c.Check(kA, Equals, nsds.ConvertKey(kB)) } }
// Query implements Query, inverting keys on the way back out. func (d *datastore) Query(q dsq.Query) (dsq.Results, error) { qr, err := d.raw.Query(q) if err != nil { return nil, err } ch := make(chan dsq.Result) go func() { defer close(ch) defer qr.Close() for r := range qr.Next() { if r.Error != nil { ch <- r continue } k := ds.NewKey(r.Entry.Key) if !d.prefix.IsAncestorOf(k) { continue } r.Entry.Key = d.Datastore.InvertKey(k).String() ch <- r } }() return dsq.DerivedResults(qr, ch), nil }
// Query implements Datastore.Query func (d *Datastore) Query(q query.Query) (query.Results, error) { results := make(chan query.Result) walkFn := func(path string, info os.FileInfo, err error) error { // remove ds path prefix if strings.HasPrefix(path, d.path) { path = path[len(d.path):] } if !info.IsDir() { if strings.HasSuffix(path, ObjectKeySuffix) { path = path[:len(path)-len(ObjectKeySuffix)] } key := ds.NewKey(path) entry := query.Entry{Key: key.String(), Value: query.NotFetched} results <- query.Result{Entry: entry} } return nil } go func() { filepath.Walk(d.path, walkFn) close(results) }() r := query.ResultsWithChan(q, results) r = query.NaiveQueryApply(q, r) return r, nil }
func strsToKeys(strs []string) []ds.Key { keys := make([]ds.Key, len(strs)) for i, s := range strs { keys[i] = ds.NewKey(s) } return keys }
// ConvertKey returns a B58 encoded Datastore key // TODO: this is hacky because it encodes every path component. some // path components may be proper strings already... func (b58KeyConverter) ConvertKey(dsk ds.Key) ds.Key { k := ds.NewKey("/") for _, n := range dsk.Namespaces() { k = k.ChildString(b58.Encode([]byte(n))) } return k }
func TestDatastoreGetNotAllowedAfterClose(t *testing.T) { t.Parallel() path := testRepoPath("test", t) assert.True(!IsInitialized(path), t, "should NOT be initialized") assert.Nil(Init(path, &config.Config{}), t, "should initialize successfully") r, err := Open(path) assert.Nil(err, t, "should open successfully") k := "key" data := []byte(k) assert.Nil(r.Datastore().Put(datastore.NewKey(k), data), t, "Put should be successful") assert.Nil(r.Close(), t) _, err = r.Datastore().Get(datastore.NewKey(k)) assert.Err(err, t, "after closer, Get should be fail") }
func (mt *mountBatch) lookupBatch(key ds.Key) (ds.Batch, ds.Key, error) { mt.lk.Lock() defer mt.lk.Unlock() child, loc, rest := mt.d.lookup(key) t, ok := mt.mounts[loc.String()] if !ok { bds, ok := child.(ds.Batching) if !ok { return nil, ds.NewKey(""), ds.ErrBatchUnsupported } var err error t, err = bds.Batch() if err != nil { return nil, ds.NewKey(""), err } mt.mounts[loc.String()] = t } return t, rest, nil }
func Example() { mp := ds.NewMapDatastore() ns := nsds.Wrap(mp, ds.NewKey("/foo/bar")) k := ds.NewKey("/beep") v := "boop" ns.Put(k, v) fmt.Printf("ns.Put %s %s\n", k, v) v2, _ := ns.Get(k) fmt.Printf("ns.Get %s -> %s\n", k, v2) k3 := ds.NewKey("/foo/bar/beep") v3, _ := mp.Get(k3) fmt.Printf("mp.Get %s -> %s\n", k3, v3) // Output: // ns.Put /beep boop // ns.Get /beep -> boop // mp.Get /foo/bar/beep -> boop }
func addTestCases(t *testing.T, d Datastore, testcases map[string]string) { for k, v := range testcases { dsk := ds.NewKey(k) if err := d.Put(dsk, []byte(v)); err != nil { t.Fatal(err) } } for k, v := range testcases { dsk := ds.NewKey(k) v2, err := d.Get(dsk) if err != nil { t.Fatal(err) } v2b := v2.([]byte) if string(v2b) != v { t.Errorf("%s values differ: %s != %s", k, v, v2) } } }
func TestDatastorePersistsFromRepoToRepo(t *testing.T) { t.Parallel() path := testRepoPath("test", t) assert.Nil(Init(path, &config.Config{}), t) r1, err := Open(path) assert.Nil(err, t) k := "key" expected := []byte(k) assert.Nil(r1.Datastore().Put(datastore.NewKey(k), expected), t, "using first repo, Put should be successful") assert.Nil(r1.Close(), t) r2, err := Open(path) assert.Nil(err, t) v, err := r2.Datastore().Get(datastore.NewKey(k)) assert.Nil(err, t, "using second repo, Get should be successful") actual, ok := v.([]byte) assert.True(ok, t, "value should be the []byte from r1's Put") assert.Nil(r2.Close(), t) assert.True(bytes.Compare(expected, actual) == 0, t, "data should match") }
func RunBatchTest(t *testing.T, ds dstore.BatchingDatastore) { batch, err := ds.Batch() if err != nil { t.Fatal(err) } r := rand.New() var blocks [][]byte var keys []dstore.Key for i := 0; i < 20; i++ { blk := make([]byte, 256*1024) r.Read(blk) blocks = append(blocks, blk) key := dstore.NewKey(base32.StdEncoding.EncodeToString(blk[:8])) keys = append(keys, key) err := batch.Put(key, blk) if err != nil { t.Fatal(err) } } // Ensure they are not in the datastore before comitting for _, k := range keys { _, err := ds.Get(k) if err == nil { t.Fatal("should not have found this block") } } // commit, write them to the datastore err = batch.Commit() if err != nil { t.Fatal(err) } for i, k := range keys { blk, err := ds.Get(k) if err != nil { t.Fatal(err) } if !bytes.Equal(blk.([]byte), blocks[i]) { t.Fatal("blocks not correct!") } } }
func TestPutGetBytes(t *testing.T) { client := clientOrAbort(t) ds, err := NewDatastore(client) if err != nil { t.Fatal(err) } key, val := datastore.NewKey("foo"), []byte("bar") dstest.Nil(ds.Put(key, val), t) v, err := ds.Get(key) if err != nil { t.Fatal(err) } if bytes.Compare(v.([]byte), val) != 0 { t.Fail() } }
func (d *datastore) runQuery(worker goprocess.Process, qrb *dsq.ResultBuilder) { var rnge *util.Range if qrb.Query.Prefix != "" { rnge = util.BytesPrefix([]byte(qrb.Query.Prefix)) } i := d.DB.NewIterator(rnge, nil) defer i.Release() // advance iterator for offset if qrb.Query.Offset > 0 { for j := 0; j < qrb.Query.Offset; j++ { i.Next() } } // iterate, and handle limit, too for sent := 0; i.Next(); sent++ { // end early if we hit the limit if qrb.Query.Limit > 0 && sent >= qrb.Query.Limit { break } k := ds.NewKey(string(i.Key())).String() e := dsq.Entry{Key: k} if !qrb.Query.KeysOnly { buf := make([]byte, len(i.Value())) copy(buf, i.Value()) e.Value = buf } select { case qrb.Output <- dsq.Result{Entry: e}: // we sent it out case <-worker.Closing(): // client told us to end early. break } } if err := i.Error(); err != nil { select { case qrb.Output <- dsq.Result{Error: err}: // client read our error case <-worker.Closing(): // client told us to end. return } } }
func TestDelete(t *testing.T) { client := clientOrAbort(t) ds, err := NewDatastore(client) if err != nil { t.Fatal(err) } key, val := datastore.NewKey("foo"), []byte("bar") dstest.Nil(ds.Put(key, val), t) dstest.Nil(ds.Delete(key), t) hasAfterDelete, err := ds.Has(key) if err != nil { t.Fatal(err) } if hasAfterDelete { t.Fail() } }
// PrefixTransform constructs a KeyTransform with a pair of functions that // add or remove the given prefix key. // // Warning: will panic if prefix not found when it should be there. This is // to avoid insidious data inconsistency errors. func PrefixTransform(prefix ds.Key) ktds.KeyTransform { return &ktds.Pair{ // Convert adds the prefix Convert: func(k ds.Key) ds.Key { return prefix.Child(k) }, // Invert removes the prefix. panics if prefix not found. Invert: func(k ds.Key) ds.Key { if !prefix.IsAncestorOf(k) { fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k) panic("expected prefix not found") } s := strings.TrimPrefix(k.String(), prefix.String()) return ds.NewKey(s) }, } }
func TestExpiry(t *testing.T) { ttl := 1 * time.Second client := clientOrAbort(t) ds, err := NewExpiringDatastore(client, ttl) if err != nil { t.Fatal(err) } key, val := datastore.NewKey("foo"), []byte("bar") dstest.Nil(ds.Put(key, val), t) time.Sleep(ttl + 1*time.Second) dstest.Nil(ds.Delete(key), t) hasAfterExpiration, err := ds.Has(key) if err != nil { t.Fatal(err) } if hasAfterExpiration { t.Fail() } }
// Query implements Query, inverting keys on the way back out. func (d *ktds) Query(q dsq.Query) (dsq.Results, error) { qr, err := d.child.Query(q) if err != nil { return nil, err } ch := make(chan dsq.Result) go func() { defer close(ch) defer qr.Close() for r := range qr.Next() { if r.Error == nil { r.Entry.Key = d.InvertKey(ds.NewKey(r.Entry.Key)).String() } ch <- r } }() return dsq.DerivedResults(qr, ch), nil }
func RunBatchDeleteTest(t *testing.T, ds dstore.BatchingDatastore) { r := rand.New() var keys []dstore.Key for i := 0; i < 20; i++ { blk := make([]byte, 16) r.Read(blk) key := dstore.NewKey(base32.StdEncoding.EncodeToString(blk[:8])) keys = append(keys, key) err := ds.Put(key, blk) if err != nil { t.Fatal(err) } } batch, err := ds.Batch() if err != nil { t.Fatal(err) } for _, k := range keys { err := batch.Delete(k) if err != nil { t.Fatal(err) } } err = batch.Commit() if err != nil { t.Fatal(err) } for _, k := range keys { _, err := ds.Get(k) if err == nil { t.Fatal("shouldnt have found block") } } }
func (d *Datastore) Query(q query.Query) (query.Results, error) { if len(q.Filters) > 0 || len(q.Orders) > 0 || q.Limit > 0 || q.Offset > 0 { // TODO this is overly simplistic, but the only caller is // `ipfs refs local` for now, and this gets us moving. return nil, errors.New("mount only supports listing all prefixed keys in random order") } key := ds.NewKey(q.Prefix) cds, mount, k := d.lookup(key) if cds == nil { return nil, errors.New("mount only supports listing a mount point") } // TODO support listing cross mount points too // delegate the query to the mounted datastore, while adjusting // keys in and out q2 := q q2.Prefix = k.String() wrapDS := keytransform.Wrap(cds, &keytransform.Pair{ Convert: func(ds.Key) ds.Key { panic("this should never be called") }, Invert: func(k ds.Key) ds.Key { return mount.Child(k) }, }) r, err := wrapDS.Query(q2) if err != nil { return nil, err } r = query.ResultsReplaceQuery(r, q) return r, nil }
func TestHasBytes(t *testing.T) { client := clientOrAbort(t) ds, err := NewDatastore(client) if err != nil { t.Fatal(err) } key, val := datastore.NewKey("foo"), []byte("bar") has, err := ds.Has(key) if err != nil { t.Fatal(err) } if has { t.Fail() } assert.Nil(ds.Put(key, val), t) hasAfterPut, err := ds.Has(key) if err != nil { t.Fatal(err) } if !hasAfterPut { t.Fail() } }
"encoding/json" "errors" "fmt" "sync" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" nsds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" key "github.com/ipfs/go-ipfs/blocks/key" "github.com/ipfs/go-ipfs/blocks/set" mdag "github.com/ipfs/go-ipfs/merkledag" "github.com/ipfs/go-ipfs/util" ) var log = util.Logger("pin") var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys") var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys") var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys") type PinMode int const ( Recursive PinMode = iota Direct Indirect NotPinned ) type Pinner interface { IsPinned(key.Key) bool Pin(context.Context, *mdag.Node, bool) error
// AllKeysChan runs a query for keys from the blockstore. // this is very simplistic, in the future, take dsq.Query as a param? // // AllKeysChan respects context func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) { // KeysOnly, because that would be _a lot_ of data. q := dsq.Query{KeysOnly: true} // datastore/namespace does *NOT* fix up Query.Prefix q.Prefix = BlockPrefix.String() res, err := bs.datastore.Query(q) if err != nil { return nil, err } // this function is here to compartmentalize get := func() (k key.Key, ok bool) { select { case <-ctx.Done(): return k, false case e, more := <-res.Next(): if !more { return k, false } if e.Error != nil { log.Debug("blockstore.AllKeysChan got err:", e.Error) return k, false } // need to convert to key.Key using key.KeyFromDsKey. k = key.KeyFromDsKey(ds.NewKey(e.Key)) log.Debug("blockstore: query got key", k) // key must be a multihash. else ignore it. _, err := mh.Cast([]byte(k)) if err != nil { return "", true } return k, true } } output := make(chan key.Key) go func() { defer func() { res.Process().Close() // ensure exit (signals early exit, too) close(output) }() for { k, ok := get() if !ok { return } if k == "" { continue } select { case <-ctx.Done(): return case output <- k: } } }() return output, nil }
"errors" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace" dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" blocks "github.com/ipfs/go-ipfs/blocks" key "github.com/ipfs/go-ipfs/blocks/key" eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog" ) var log = eventlog.Logger("blockstore") // BlockPrefix namespaces blockstore datastores var BlockPrefix = ds.NewKey("blocks") var ValueTypeMismatch = errors.New("The retrieved value is not a Block") var ErrNotFound = errors.New("blockstore: block not found") // Blockstore wraps a ThreadSafeDatastore type Blockstore interface { DeleteBlock(key.Key) error Has(key.Key) (bool, error) Get(key.Key) (*blocks.Block, error) Put(*blocks.Block) error AllKeysChan(ctx context.Context) (<-chan key.Key, error) }
func TestCoalesceDelete(t *testing.T) { m := setup() done := make(chan struct{}) errs := make(chan error, 30) m.ds.Put(ds.NewKey("foo1"), "bar1") m.ds.Put(ds.NewKey("foo2"), "bar2") m.ds.Put(ds.NewKey("foo3"), "bar3") for i := 0; i < 10; i++ { go func() { err := m.ds.Delete(ds.NewKey("foo1")) if err != nil { errs <- err } has, err := m.ds.Has(ds.NewKey("foo1")) if err != nil { errs <- err } if has { t.Error("still have it after deleting") } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { err := m.ds.Delete(ds.NewKey("foo2")) if err != nil { errs <- err } has, err := m.ds.Has(ds.NewKey("foo2")) if err != nil { errs <- err } if has { t.Error("still have it after deleting") } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { has, err := m.ds.Has(ds.NewKey("foo3")) if err != nil { errs <- err } if !has { t.Error("should still have foo3") } done <- struct{}{} }() } for i := 0; i < 10; i++ { go func() { has, err := m.ds.Has(ds.NewKey("foo4")) if err != nil { errs <- err } if has { t.Error("should not have foo4") } done <- struct{}{} }() } for i := 0; i < 40; i++ { <-done } if m.inside != 9 { t.Error("incalls should be 9", m.inside) } if m.outside != 63 { t.Error("outcalls should be 63", m.outside) } }