func (store *BoltStore) size(bucket *bolt.Bucket, key string, size *AtomicInt64) int64 {
var statics *AtomicInt64 = size
if statics == nil {
statics = NewAtomicInt64(0)
}
if key != "" {
b := bucket.Get([]byte(key))
if b != nil {
return statics.Caculate(int64(len(key) + len(b)))
}
bucket = bucket.Bucket([]byte(key))
}
if bucket != nil {
c := bucket.Cursor()
// to do staticsing
for k, v := c.First(); k != nil; k, v = c.Next() {
if v != nil {
statics.Caculate(int64(len(k) + len(v)))
} else {
store.size(bucket, string(k), statics)
}
}
statics.Caculate(int64(len(key)))
return statics.Value()
}
//! null
return statics.Value()
}
func (store *BoltStore) Get(uri string, nested bool) (interface{}, error) {
namespace, key := util.NamespaceKey(uri)
bucketkeys := util.PathFolders(namespace)
var bucket *bolt.Bucket = nil
var result interface{}
e := store.db.View(func(tx *bolt.Tx) error {
for _, bucketkey := range bucketkeys {
if bucket == nil {
bucket = tx.Bucket([]byte(bucketkey))
if bucket == nil {
return errors.New(uri + " not exists")
}
} else {
bucket = bucket.Bucket([]byte(bucketkey))
if bucket == nil {
return errors.New(uri + " not exists")
}
}
}
if bucket == nil {
return errors.New(uri + " not exists")
}
val, err := store.get(bucket, key, nested)
if err != nil {
return err
}
result = val
return nil
})
return result, e
}
func (store *BoltStore) get(bucket *bolt.Bucket, key string, nested bool) (interface{}, error) {
if key != "" {
b := bucket.Get([]byte(key))
if b != nil {
return store.coder.Decode(b)
}
bucket = bucket.Bucket([]byte(key))
}
if bucket != nil {
m := make(map[string]interface{})
c := bucket.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v != nil {
vv, err := store.coder.Decode(v)
if err != nil {
return nil, err
}
m[string(k)] = vv
}
if v == nil && nested == true {
vv, err := store.get(bucket, string(k), nested)
if err != nil {
return nil, err
}
m[string(k)] = vv
}
}
return m, nil
}
return nil, fmt.Errorf("bucket not exist")
}
func (store *BoltStore) MultiGet(uri string, suffix string, nested bool) ([]interface{}, error) {
results := []interface{}{}
bucketkeys := util.PathFolders(uri)
var bucket *bolt.Bucket = nil
e := store.db.View(func(tx *bolt.Tx) error {
for _, bucketkey := range bucketkeys {
if bucket == nil {
bucket = tx.Bucket([]byte(bucketkey))
if bucket == nil {
return errors.New(uri + " not exists")
}
} else {
bucket = bucket.Bucket([]byte(bucketkey))
if bucket == nil {
return errors.New(uri + " not exists")
}
}
}
if bucket == nil {
return errors.New(uri + " not exists")
}
vals, err := store.multi_get(bucket, suffix, nested)
if err != nil {
return err
}
results = append(results, vals...)
return nil
})
return results, e
}
func (store *BoltStore) Size(uri string) (int64, error) {
namespace, key := util.NamespaceKey(uri)
bucketkeys := util.PathFolders(namespace)
var bucket *bolt.Bucket = nil
var result int64 = 0
e := store.db.View(func(tx *bolt.Tx) error {
for _, bucketkey := range bucketkeys {
if bucket == nil {
bucket = tx.Bucket([]byte(bucketkey))
if bucket == nil {
return errors.New(uri + " not exists")
}
} else {
bucket = bucket.Bucket([]byte(bucketkey))
if bucket == nil {
return errors.New(uri + " not exists")
}
}
}
if bucket == nil {
return errors.New(uri + " not exists")
}
result = store.size(bucket, key, nil)
return nil
})
return result, e
}
// Process verifies the signature and uniqueness of the random roken.
func (MailgunValidator) Process(h Hook, r Request, b *bolt.Bucket) error {
// Check HMAC
apikey := b.Get([]byte(fmt.Sprintf("%s-apikey", h.ID)))
if apikey == nil {
return errors.New("mailgun validator not initialized")
}
if r.Headers["Content-Type"] != "application/x-www-form-urlencoded" {
return fmt.Errorf("unexpected Content-Type: %q", r.Headers["Content-Type"])
}
form, err := url.ParseQuery(string(r.Body))
if err != nil {
return fmt.Errorf("error parsing request body: %s", err)
}
timestamp := form.Get("timestamp")
signature := []byte(form.Get("signature"))
token := form.Get("token")
mac := hmac.New(sha256.New, apikey)
mac.Write([]byte(timestamp + token))
expected := []byte(hex.EncodeToString(mac.Sum(nil)))
if !hmac.Equal(signature, expected) {
return errors.New("invalid signature")
}
// Check uniqueness
tokens := b.Bucket([]byte("tokens"))
if p := tokens.Get([]byte(token)); p != nil {
return errors.New("duplicate request token received")
}
return tokens.Put([]byte(token), []byte{})
}
func (self *TX) dig(keys [][]byte, create bool) (buckets []*bolt.Bucket, err error) {
var bucket *bolt.Bucket
if create {
if bucket, err = self.tx.CreateBucketIfNotExists(keys[0]); err != nil {
return
}
} else {
if bucket = self.tx.Bucket(keys[0]); bucket == nil {
err = ErrNotFound
return
}
}
buckets = append(buckets, bucket)
for keys = keys[1:]; len(keys) > 0; keys = keys[1:] {
if create {
if bucket, err = bucket.CreateBucketIfNotExists(keys[0]); err != nil {
return
}
} else {
if bucket = bucket.Bucket(keys[0]); bucket == nil {
err = ErrNotFound
return
}
}
buckets = append(buckets, bucket)
}
return
}
func (store *BoltStore) multi_get(bucket *bolt.Bucket, suffix string, nested bool) ([]interface{}, error) {
results := []interface{}{}
if bucket != nil {
c := bucket.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if suffix == "" {
if v != nil {
vv, err := store.coder.Decode(v)
if err != nil {
return nil, err
}
results = append(results, vv)
continue
}
if nested {
bucket_nested := bucket.Bucket(k)
if bucket_nested != nil {
nests, err := store.multi_get(bucket_nested, suffix, nested)
if err != nil {
return results, err
}
results = append(results, nests...)
}
}
} else {
if strings.HasSuffix(string(k), suffix) {
vv, err := store.get(bucket, string(k), true)
if err != nil {
return results, err
}
results = append(results, vv)
continue
} else {
if nested && v == nil {
bucket_nested := bucket.Bucket(k)
if bucket_nested != nil {
nests, err := store.multi_get(bucket_nested, suffix, nested)
if err != nil {
return results, err
}
results = append(results, nests...)
}
}
}
}
}
}
return results, nil
}
func bucketFor(key Key, b *bolt.Bucket) (*bolt.Bucket, []byte, error) {
var err error
elems, last := key.B()
for _, e := range elems {
// CreateBucketIfNotExists requires a writeable transaction.
if new := b.Bucket(e); new != nil {
b = new
continue
}
if b, err = b.CreateBucket(e); err != nil {
return b, last, err
}
}
return b, last, nil
}
func walkBucket(parent *bolt.Bucket, k []byte, v []byte, w io.Writer) error {
if _, err := fmt.Fprintf(w, "%d:%x=%x\n", parent.Sequence(), k, v); err != nil {
return err
}
// not a bucket, exit.
if v != nil {
return nil
}
return parent.ForEach(func(k, v []byte) error {
if v == nil {
return walkBucket(parent.Bucket(k), k, nil, w)
}
return walkBucket(parent, k, v, w)
})
}
// GetBucket returns the given bucket below the current node.
func (n *node) GetBucket(tx *bolt.Tx, children ...string) *bolt.Bucket {
var b *bolt.Bucket
bucketNames := append(n.rootBucket, children...)
for _, bucketName := range bucketNames {
if b != nil {
if b = b.Bucket([]byte(bucketName)); b == nil {
return nil
}
} else {
if b = tx.Bucket([]byte(bucketName)); b == nil {
return nil
}
}
}
return b
}
// NewUniqueIndex loads a UniqueIndex
func NewUniqueIndex(parent *bolt.Bucket, indexName []byte) (*UniqueIndex, error) {
var err error
b := parent.Bucket(indexName)
if b == nil {
if !parent.Writable() {
return nil, ErrNotFound
}
b, err = parent.CreateBucket(indexName)
if err != nil {
return nil, err
}
}
return &UniqueIndex{
IndexBucket: b,
Parent: parent,
}, nil
}
func readBucket(b *bolt.Bucket) (*BoltBucket, error) {
bb := new(BoltBucket)
b.ForEach(func(k, v []byte) error {
if v == nil {
tb, err := readBucket(b.Bucket(k))
tb.parent = bb
if err == nil {
tb.name = string(k)
bb.buckets = append(bb.buckets, *tb)
}
} else {
tp := BoltPair{key: string(k), val: string(v)}
tp.parent = bb
bb.pairs = append(bb.pairs, tp)
}
return nil
})
return bb, nil
}
func (cmd *CompactCommand) walkBucket(b *bolt.Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error {
// Execute callback.
if err := fn(keypath, k, v, seq); err != nil {
return err
}
// If this is not a bucket then stop.
if v != nil {
return nil
}
// Iterate over each child key/value.
keypath = append(keypath, k)
return b.ForEach(func(k, v []byte) error {
if v == nil {
bkt := b.Bucket(k)
return cmd.walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn)
}
return cmd.walkBucket(b, keypath, k, v, b.Sequence(), fn)
})
}
func (b *Bucket) fill(bucket *bolt.Bucket) {
//fmt.Println("Fill bucket ", bucket.Root())
//fill subbuckets
// bucket.Tx().ForEach(func(name []byte, buck *bolt.Bucket) error {
// fmt.Println("subbucket ", string(name))
// subbuck := Bucket{
// Name: string(name),
// }
// fmt.Println("start fill ", string(name))
// subbuck.fill(buck)
// b.Subbuckets = append(b.Subbuckets, subbuck)
// return nil
// })
//fill entries
bucket.ForEach(func(k, v []byte) error {
if len(v) == 0 { //subbucket
sb := bucket.Bucket(k)
if sb == nil {
b.Entries = append(b.Entries, Entry{string(k), string(v)})
return nil
}
subbuck := Bucket{
Name: string(k),
Subbuckets: []Bucket{},
Entries: []Entry{},
}
subbuck.fill(sb)
b.Subbuckets = append(b.Subbuckets, subbuck)
} else {
b.Entries = append(b.Entries, decodeEntry(k, v))
}
return nil
})
}
func getBucketContent(b *bolt.Bucket) (content map[string]string, nested []string) {
content = map[string]string{}
nested = []string{}
b.ForEach(func(k, v []byte) error {
if len(v) != 0 {
var out bytes.Buffer
json.Indent(&out, v, "", "\t")
content[string(k)] = out.String()
return nil
}
// may be k is a bucket
nb := b.Bucket(k)
if nb != nil {
nested = append(nested, string(k))
} else {
content[string(k)] = ""
}
return nil
})
return content, nested
}
// fetchNodeChannels retrieves all active channels from the target
// nodeChanBucket. This function is typically used to fetch all the active
// channels related to a particualr node.
func (d *DB) fetchNodeChannels(openChanBucket,
nodeChanBucket *bolt.Bucket) ([]*OpenChannel, error) {
var channels []*OpenChannel
// Once we have the node's channel bucket, iterate through each
// item in the inner chan ID bucket. This bucket acts as an
// index for all channels we currently have open with this node.
nodeChanIDBucket := nodeChanBucket.Bucket(chanIDBucket[:])
if nodeChanIDBucket == nil {
return nil, nil
}
err := nodeChanIDBucket.ForEach(func(k, v []byte) error {
if k == nil {
return nil
}
outBytes := bytes.NewReader(k)
chanID := &wire.OutPoint{}
if err := readOutpoint(outBytes, chanID); err != nil {
return err
}
oChannel, err := fetchOpenChannel(openChanBucket,
nodeChanBucket, chanID)
if err != nil {
return err
}
oChannel.Db = d
channels = append(channels, oChannel)
return nil
})
if err != nil {
return nil, err
}
return channels, nil
}
// Process verifies the signature and uniqueness of the delivery identifier.
func (GithubValidator) Process(h Hook, r Request, b *bolt.Bucket) error {
// Check HMAC
secret := b.Get([]byte(fmt.Sprintf("%s-secret", h.ID)))
if secret == nil {
return errors.New("github validator not initialized")
}
signature := []byte(r.Headers["X-Hub-Signature"])
mac := hmac.New(sha1.New, secret)
mac.Write(bytes.TrimSpace(r.Body))
expected := append([]byte("sha1="), hex.EncodeToString(mac.Sum(nil))...)
if !hmac.Equal(signature, expected) {
return errors.New("invalid signature")
}
// Check uniqueness
id := []byte(r.Headers["X-Github-Delivery"])
deliveries := b.Bucket([]byte("deliveries"))
if did := deliveries.Get([]byte(id)); did != nil {
return errors.New("duplicate delivery")
}
return deliveries.Put([]byte(id), []byte{})
}
// Process makes sure incoming requests do not exceed the configured rate
// limit.
func (RateLimitFilter) Process(h Hook, r Request, b *bolt.Bucket) error {
amount, _ := strconv.Atoi(string(b.Get([]byte(fmt.Sprintf("%s-amount", h.ID)))))
interval, _ := strconv.Atoi(string(b.Get([]byte(fmt.Sprintf("%s-interval", h.ID)))))
if amount <= 0 || interval <= 0 {
return errors.New("rate limit filter not initialized")
}
b = b.Bucket([]byte("requests"))
// store current timestamp
now := time.Now()
k := []byte(fmt.Sprintf("%d", now.UnixNano()))
if err := b.Put(k, nil); err != nil {
return err
}
// count requests
c := b.Cursor()
from := []byte(fmt.Sprintf("%d", now.Add(time.Duration(-interval)*time.Second).UnixNano()))
var count int
for k, _ := c.Seek(from); k != nil; k, _ = c.Next() {
count++
}
if count > amount {
return fmt.Errorf("rate limit exceeded (limit=%d count=%d)", amount, count)
}
// cleanup old entries
for k, _ := c.First(); k != nil && bytes.Compare(k, from) <= 0; k, _ = c.Next() {
if err := b.Delete(k); err != nil {
return err
}
}
return nil
}
func exportBucket(b *bolt.Bucket) (*rawMessage, error) {
// Encode individual key/value pairs into raw messages.
var children = make([]*rawMessage, 0)
err := b.ForEach(func(k, v []byte) error {
var err error
// If there is no value then it is a bucket.
if v == nil {
child, err := exportBucket(b.Bucket(k))
if err != nil {
return fmt.Errorf("bucket: %s", err)
}
child.Key = k
children = append(children, child)
return nil
}
// Otherwise it's a regular key.
var child = &rawMessage{Key: k}
if child.Value, err = json.Marshal(v); err != nil {
return fmt.Errorf("value: %s", err)
}
children = append(children, child)
return nil
})
if err != nil {
return nil, err
}
// Encode bucket into a raw message.
var root = rawMessage{Type: "bucket"}
if root.Value, err = json.Marshal(children); err != nil {
return nil, fmt.Errorf("children: %s", err)
}
return &root, nil
}
func newMeta(b *bolt.Bucket, n Node) (*meta, error) {
m := b.Bucket([]byte(metadataBucket))
if m != nil {
name := m.Get([]byte(metaCodec))
if string(name) != n.Codec().Name() {
return nil, ErrDifferentCodec
}
return &meta{
node: n,
bucket: m,
}, nil
}
m, err := b.CreateBucket([]byte(metadataBucket))
if err != nil {
return nil, err
}
m.Put([]byte(metaCodec), []byte(n.Codec().Name()))
return &meta{
node: n,
bucket: m,
}, nil
}
// NewListIndex loads a ListIndex
func NewListIndex(parent *bolt.Bucket, indexName []byte) (*ListIndex, error) {
var err error
b := parent.Bucket(indexName)
if b == nil {
if !parent.Writable() {
return nil, ErrNotFound
}
b, err = parent.CreateBucket(indexName)
if err != nil {
return nil, err
}
}
ids, err := NewUniqueIndex(b, []byte("storm__ids"))
if err != nil {
return nil, err
}
return &ListIndex{
IndexBucket: b,
Parent: parent,
IDs: ids,
}, nil
}
func openTabAll(bktBook *bolt.Bucket, tabid string) (*bolt.Bucket, *bolt.Bucket) {
bktTab := bktBook.Bucket(bs("tab_" + tabid))
bktNotes := bktTab.Bucket(bs("notes"))
bktPrevids := bktTab.Bucket(bs("previds"))
return bktNotes, bktPrevids
}
func openPrevids(bktTab *bolt.Bucket) *bolt.Bucket {
return bktTab.Bucket(bs("previds"))
}
func openNotes(bktTab *bolt.Bucket) *bolt.Bucket {
return bktTab.Bucket(bs("notes"))
}
func openTab(bktBook *bolt.Bucket, tabid string) *bolt.Bucket {
return bktBook.Bucket(bs("tab_" + tabid))
}
func openTabs(bktBook *bolt.Bucket) *bolt.Bucket {
return bktBook.Bucket(bs("tabs"))
}
