func (ks *DSSuite) TestBasic(c *C) {
var size = 1000
d, err := lru.NewDatastore(size)
c.Check(err, Equals, nil)
for i := 0; i < size; i++ {
err := d.Put(ds.NewKey(strconv.Itoa(i)), i)
c.Check(err, Equals, nil)
}
for i := 0; i < size; i++ {
j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
c.Check(j, Equals, i)
c.Check(err, Equals, nil)
}
for i := 0; i < size; i++ {
err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i)
c.Check(err, Equals, nil)
}
for i := 0; i < size; i++ {
j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
c.Check(j, Equals, nil)
c.Check(err, Equals, ds.ErrNotFound)
}
for i := 0; i < size; i++ {
j, err := d.Get(ds.NewKey(strconv.Itoa(i + size)))
c.Check(j, Equals, i)
c.Check(err, Equals, nil)
}
}
func (ks *DSSuite) TestBasic(c *C) {
mpds := ds.NewMapDatastore()
nsds := ns.Wrap(mpds, ds.NewKey("abc"))
keys := strsToKeys([]string{
"foo",
"foo/bar",
"foo/bar/baz",
"foo/barb",
"foo/bar/bazb",
"foo/bar/baz/barb",
})
for _, k := range keys {
err := nsds.Put(k, []byte(k.String()))
c.Check(err, Equals, nil)
}
for _, k := range keys {
v1, err := nsds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
r, err := d.Query(q)
c.Check(err, Equals, nil)
e, err := r.Rest()
c.Check(err, Equals, nil)
return ds.EntryKeys(e)
}
listA := run(mpds, dsq.Query{})
listB := run(nsds, dsq.Query{})
c.Check(len(listA), Equals, len(listB))
// sort them cause yeah.
sort.Sort(ds.KeySlice(listA))
sort.Sort(ds.KeySlice(listB))
for i, kA := range listA {
kB := listB[i]
c.Check(nsds.InvertKey(kA), Equals, kB)
c.Check(kA, Equals, nsds.ConvertKey(kB))
}
}
// Query implements Datastore.Query
func (d *Datastore) Query(q query.Query) (query.Results, error) {
results := make(chan query.Result)
walkFn := func(path string, info os.FileInfo, err error) error {
// remove ds path prefix
if strings.HasPrefix(path, d.path) {
path = path[len(d.path):]
}
if !info.IsDir() {
if strings.HasSuffix(path, ObjectKeySuffix) {
path = path[:len(path)-len(ObjectKeySuffix)]
}
key := ds.NewKey(path)
entry := query.Entry{Key: key.String(), Value: query.NotFetched}
results <- query.Result{Entry: entry}
}
return nil
}
go func() {
filepath.Walk(d.path, walkFn)
close(results)
}()
r := query.ResultsWithChan(q, results)
r = query.NaiveQueryApply(q, r)
return r, nil
}
// Query implements Query, inverting keys on the way back out.
func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
qr, err := d.raw.Query(q)
if err != nil {
return nil, err
}
ch := make(chan dsq.Result)
go func() {
defer close(ch)
defer qr.Close()
for r := range qr.Next() {
if r.Error != nil {
ch <- r
continue
}
k := ds.NewKey(r.Entry.Key)
if !d.prefix.IsAncestorOf(k) {
continue
}
r.Entry.Key = d.Datastore.InvertKey(k).String()
ch <- r
}
}()
return dsq.DerivedResults(qr, ch), nil
}
func strsToKeys(strs []string) []ds.Key {
keys := make([]ds.Key, len(strs))
for i, s := range strs {
keys[i] = ds.NewKey(s)
}
return keys
}
func transferBlocks(from, to dstore.Datastore, fpref, tpref string, verbose bool) error {
q := dsq.Query{Prefix: fpref, KeysOnly: true}
res, err := from.Query(q)
if err != nil {
return err
}
showProgress := func(i int) {}
if verbose {
showProgress = func(i int) {
fmt.Printf("\rmoving objects: %d", i)
}
defer fmt.Println("")
}
i := 0
for result := range res.Next() {
i++
showProgress(i)
nkey := fmt.Sprintf("%s%s", tpref, result.Key[len(fpref):])
fkey := dstore.NewKey(result.Key)
val, err := from.Get(fkey)
if err != nil {
return err
}
err = to.Put(dstore.NewKey(nkey), val)
if err != nil {
return err
}
err = from.Delete(fkey)
if err != nil {
return err
}
}
return nil
}
func Example() {
mp := ds.NewMapDatastore()
ns := nsds.Wrap(mp, ds.NewKey("/foo/bar"))
k := ds.NewKey("/beep")
v := "boop"
ns.Put(k, v)
fmt.Printf("ns.Put %s %s\n", k, v)
v2, _ := ns.Get(k)
fmt.Printf("ns.Get %s -> %s\n", k, v2)
k3 := ds.NewKey("/foo/bar/beep")
v3, _ := mp.Get(k3)
fmt.Printf("mp.Get %s -> %s\n", k3, v3)
// Output:
// ns.Put /beep boop
// ns.Get /beep -> boop
// mp.Get /foo/bar/beep -> boop
}
func (d *datastore) runQuery(worker goprocess.Process, qrb *dsq.ResultBuilder) {
var rnge *util.Range
if qrb.Query.Prefix != "" {
rnge = util.BytesPrefix([]byte(qrb.Query.Prefix))
}
i := d.DB.NewIterator(rnge, nil)
defer i.Release()
// advance iterator for offset
if qrb.Query.Offset > 0 {
for j := 0; j < qrb.Query.Offset; j++ {
i.Next()
}
}
// iterate, and handle limit, too
for sent := 0; i.Next(); sent++ {
// end early if we hit the limit
if qrb.Query.Limit > 0 && sent >= qrb.Query.Limit {
break
}
k := ds.NewKey(string(i.Key())).String()
e := dsq.Entry{Key: k}
if !qrb.Query.KeysOnly {
buf := make([]byte, len(i.Value()))
copy(buf, i.Value())
e.Value = buf
}
select {
case qrb.Output <- dsq.Result{Entry: e}: // we sent it out
case <-worker.Closing(): // client told us to end early.
break
}
}
if err := i.Error(); err != nil {
select {
case qrb.Output <- dsq.Result{Error: err}: // client read our error
case <-worker.Closing(): // client told us to end.
return
}
}
}
// PrefixTransform constructs a KeyTransform with a pair of functions that
// add or remove the given prefix key.
//
// Warning: will panic if prefix not found when it should be there. This is
// to avoid insidious data inconsistency errors.
func PrefixTransform(prefix ds.Key) ktds.KeyTransform {
return &ktds.Pair{
// Convert adds the prefix
Convert: func(k ds.Key) ds.Key {
return prefix.Child(k)
},
// Invert removes the prefix. panics if prefix not found.
Invert: func(k ds.Key) ds.Key {
if !prefix.IsAncestorOf(k) {
fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k)
panic("expected prefix not found")
}
s := strings.TrimPrefix(k.String(), prefix.String())
return ds.NewKey(s)
},
}
}
func (ks *DSSuite) TestBasic(c *C) {
pair := &kt.Pair{
Convert: func(k ds.Key) ds.Key {
return ds.NewKey("/abc").Child(k)
},
Invert: func(k ds.Key) ds.Key {
// remove abc prefix
l := k.List()
if l[0] != "abc" {
panic("key does not have prefix. convert failed?")
}
return ds.KeyWithNamespaces(l[1:])
},
}
mpds := ds.NewMapDatastore()
ktds := kt.Wrap(mpds, pair)
keys := strsToKeys([]string{
"foo",
"foo/bar",
"foo/bar/baz",
"foo/barb",
"foo/bar/bazb",
"foo/bar/baz/barb",
})
for _, k := range keys {
err := ktds.Put(k, []byte(k.String()))
c.Check(err, Equals, nil)
}
for _, k := range keys {
v1, err := ktds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
r, err := d.Query(q)
c.Check(err, Equals, nil)
e, err := r.Rest()
c.Check(err, Equals, nil)
return ds.EntryKeys(e)
}
listA := run(mpds, dsq.Query{})
listB := run(ktds, dsq.Query{})
c.Check(len(listA), Equals, len(listB))
// sort them cause yeah.
sort.Sort(ds.KeySlice(listA))
sort.Sort(ds.KeySlice(listB))
for i, kA := range listA {
kB := listB[i]
c.Check(pair.Invert(kA), Equals, kB)
c.Check(kA, Equals, pair.Convert(kB))
}
c.Log("listA: ", listA)
c.Log("listB: ", listB)
}