// ConvertKey returns a B58 encoded Datastore key
// TODO: this is hacky because it encodes every path component. some
// path components may be proper strings already...
func (b58KeyConverter) ConvertKey(dsk ds.Key) ds.Key {
k := ds.NewKey("/")
for _, n := range dsk.Namespaces() {
k = k.ChildString(b58.Encode([]byte(n)))
}
return k
}
// Returns ErrInvalidType if value is not of type []byte.
//
// Note: using sync = false.
// see http://godoc.org/github.com/syndtr/goleveldb/leveldb/opt#WriteOptions
func (d *datastore) Put(key ds.Key, value interface{}) (err error) {
val, ok := value.([]byte)
if !ok {
return ds.ErrInvalidType
}
return d.DB.Put(key.Bytes(), val, nil)
}
func (d *datastore) Delete(key ds.Key) (err error) {
err = d.DB.Delete(key.Bytes(), nil)
if err == leveldb.ErrNotFound {
return ds.ErrNotFound
}
return err
}
func (ds *S3Datastore) Put(key datastore.Key, value interface{}) (err error) {
data, ok := value.([]byte)
if !ok {
return ErrInvalidType
}
// TODO extract perms and s3 options
return ds.Client.Bucket(ds.Bucket).Put(key.String(), data, "application/protobuf", s3.PublicRead, s3.Options{})
}
func (d *datastore) Get(key ds.Key) (value interface{}, err error) {
val, err := d.DB.Get(key.Bytes(), nil)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, ds.ErrNotFound
}
return nil, err
}
return val, nil
}
func (d *Datastore) Delete(key ds.Key) (err error) {
id := d.KeyHash(key)
res, err := core.Delete(false, d.Index(key), key.Type(), id, 0, "")
if err != nil {
return err
}
if !res.Ok {
return fmt.Errorf("Elasticsearch response: NOT OK. %v", res)
}
return nil
}
// value should be JSON serializable.
func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
id := d.KeyHash(key)
res, err := core.Index(false, d.Index(key), key.Type(), id, value)
if err != nil {
return err
}
if !res.Ok {
return fmt.Errorf("Elasticsearch response: NOT OK. %v", res)
}
return nil
}
func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
id := d.KeyHash(key)
res, err := core.Get(false, d.Index(key), key.Type(), id)
if err != nil {
return nil, err
}
if !res.Ok {
return nil, fmt.Errorf("Elasticsearch response: NOT OK. %v", res)
}
return res.Source, nil
}
func (d *Datastore) lookup(key ds.Key) (ds.Datastore, ds.Key, ds.Key) {
d.lk.Lock()
defer d.lk.Unlock()
for _, m := range d.mounts {
if m.Prefix.Equal(key) || m.Prefix.IsAncestorOf(key) {
s := strings.TrimPrefix(key.String(), m.Prefix.String())
k := ds.NewKey(s)
return m.Datastore, m.Prefix, k
}
}
return nil, ds.NewKey("/"), key
}
func (ds *Datastore) Put(key datastore.Key, value interface{}) error {
ds.mu.Lock()
defer ds.mu.Unlock()
data, ok := value.([]byte)
if !ok {
return ErrInvalidType
}
ds.client.Append("SET", key.String(), data)
if ds.ttl != 0 {
ds.client.Append("EXPIRE", key.String(), ds.ttl.Seconds())
}
if err := ds.client.GetReply().Err; err != nil {
return fmt.Errorf("failed to put value: %s", err)
}
if ds.ttl != 0 {
if err := ds.client.GetReply().Err; err != nil {
return fmt.Errorf("failed to set expiration: %s", err)
}
}
return nil
}
// PrefixTransform constructs a KeyTransform with a pair of functions that
// add or remove the given prefix key.
//
// Warning: will panic if prefix not found when it should be there. This is
// to avoid insidious data inconsistency errors.
func PrefixTransform(prefix ds.Key) ktds.KeyTransform {
return &ktds.Pair{
// Convert adds the prefix
Convert: func(k ds.Key) ds.Key {
return prefix.Child(k)
},
// Invert removes the prefix. panics if prefix not found.
Invert: func(k ds.Key) ds.Key {
if !prefix.IsAncestorOf(k) {
fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k)
panic("expected prefix not found")
}
s := strings.TrimPrefix(k.String(), prefix.String())
return ds.NewKey(s)
},
}
}
func (d *datastore) Has(key ds.Key) (exists bool, err error) {
return d.DB.Has(key.Bytes(), nil)
}
func (ds *Datastore) Get(key datastore.Key) (value interface{}, err error) {
ds.mu.Lock()
defer ds.mu.Unlock()
return ds.client.Cmd("GET", key.String()).Bytes()
}
func (ds *Datastore) Has(key datastore.Key) (exists bool, err error) {
ds.mu.Lock()
defer ds.mu.Unlock()
return ds.client.Cmd("EXISTS", key.String()).Bool()
}
func (ds *Datastore) Delete(key datastore.Key) (err error) {
ds.mu.Lock()
defer ds.mu.Unlock()
return ds.client.Cmd("DEL", key.String()).Err
}
// KeyFromDsKey returns a Datastore key
func KeyFromDsKey(dsk ds.Key) Key {
return Key(dsk.String()[1:])
}
func (ds *S3Datastore) Has(key datastore.Key) (exists bool, err error) {
return ds.Client.Bucket(ds.Bucket).Exists(key.String())
}
// Hash a key and return the first 16 hex chars of its blake2b hash.
// basically: Blake2b(key).HexString[:16]
func BlakeKeyHash(key ds.Key) string {
h := blake2.NewBlake2B()
h.Write(key.Bytes())
d := h.Sum(nil)
return fmt.Sprintf("%x", d)[:16]
}
// Returns the ElasticSearch index for given key. If the datastore specifies
// an index, use that. Else, key.Parent
func (d *Datastore) Index(key ds.Key) string {
if len(d.index) > 0 {
return d.index
}
return key.Parent().BaseNamespace()
}
func (ds *S3Datastore) Get(key datastore.Key) (value interface{}, err error) {
return ds.Client.Bucket(ds.Bucket).Get(key.String())
}
func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
id := d.KeyHash(key)
return core.Exists(false, d.Index(key), key.Type(), id)
}
func (ds *S3Datastore) Delete(key datastore.Key) (err error) {
return ds.Client.Bucket(ds.Bucket).Del(key.String())
}
// KeyFilename returns the filename associated with `key`
func (d *Datastore) KeyFilename(key ds.Key) string {
return filepath.Join(d.path, key.String(), ObjectKeySuffix)
}