func (store *BoltStore) size(bucket *bolt.Bucket, key string, size *AtomicInt64) int64 { var statics *AtomicInt64 = size if statics == nil { statics = NewAtomicInt64(0) } if key != "" { b := bucket.Get([]byte(key)) if b != nil { return statics.Caculate(int64(len(key) + len(b))) } bucket = bucket.Bucket([]byte(key)) } if bucket != nil { c := bucket.Cursor() // to do staticsing for k, v := c.First(); k != nil; k, v = c.Next() { if v != nil { statics.Caculate(int64(len(k) + len(v))) } else { store.size(bucket, string(k), statics) } } statics.Caculate(int64(len(key))) return statics.Value() } //! null return statics.Value() }
func (store *BoltStore) Get(uri string, nested bool) (interface{}, error) { namespace, key := util.NamespaceKey(uri) bucketkeys := util.PathFolders(namespace) var bucket *bolt.Bucket = nil var result interface{} e := store.db.View(func(tx *bolt.Tx) error { for _, bucketkey := range bucketkeys { if bucket == nil { bucket = tx.Bucket([]byte(bucketkey)) if bucket == nil { return errors.New(uri + " not exists") } } else { bucket = bucket.Bucket([]byte(bucketkey)) if bucket == nil { return errors.New(uri + " not exists") } } } if bucket == nil { return errors.New(uri + " not exists") } val, err := store.get(bucket, key, nested) if err != nil { return err } result = val return nil }) return result, e }
func (store *BoltStore) get(bucket *bolt.Bucket, key string, nested bool) (interface{}, error) { if key != "" { b := bucket.Get([]byte(key)) if b != nil { return store.coder.Decode(b) } bucket = bucket.Bucket([]byte(key)) } if bucket != nil { m := make(map[string]interface{}) c := bucket.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { if v != nil { vv, err := store.coder.Decode(v) if err != nil { return nil, err } m[string(k)] = vv } if v == nil && nested == true { vv, err := store.get(bucket, string(k), nested) if err != nil { return nil, err } m[string(k)] = vv } } return m, nil } return nil, fmt.Errorf("bucket not exist") }
func (store *BoltStore) MultiGet(uri string, suffix string, nested bool) ([]interface{}, error) { results := []interface{}{} bucketkeys := util.PathFolders(uri) var bucket *bolt.Bucket = nil e := store.db.View(func(tx *bolt.Tx) error { for _, bucketkey := range bucketkeys { if bucket == nil { bucket = tx.Bucket([]byte(bucketkey)) if bucket == nil { return errors.New(uri + " not exists") } } else { bucket = bucket.Bucket([]byte(bucketkey)) if bucket == nil { return errors.New(uri + " not exists") } } } if bucket == nil { return errors.New(uri + " not exists") } vals, err := store.multi_get(bucket, suffix, nested) if err != nil { return err } results = append(results, vals...) return nil }) return results, e }
func (store *BoltStore) Size(uri string) (int64, error) { namespace, key := util.NamespaceKey(uri) bucketkeys := util.PathFolders(namespace) var bucket *bolt.Bucket = nil var result int64 = 0 e := store.db.View(func(tx *bolt.Tx) error { for _, bucketkey := range bucketkeys { if bucket == nil { bucket = tx.Bucket([]byte(bucketkey)) if bucket == nil { return errors.New(uri + " not exists") } } else { bucket = bucket.Bucket([]byte(bucketkey)) if bucket == nil { return errors.New(uri + " not exists") } } } if bucket == nil { return errors.New(uri + " not exists") } result = store.size(bucket, key, nil) return nil }) return result, e }
// Process verifies the signature and uniqueness of the random roken. func (MailgunValidator) Process(h Hook, r Request, b *bolt.Bucket) error { // Check HMAC apikey := b.Get([]byte(fmt.Sprintf("%s-apikey", h.ID))) if apikey == nil { return errors.New("mailgun validator not initialized") } if r.Headers["Content-Type"] != "application/x-www-form-urlencoded" { return fmt.Errorf("unexpected Content-Type: %q", r.Headers["Content-Type"]) } form, err := url.ParseQuery(string(r.Body)) if err != nil { return fmt.Errorf("error parsing request body: %s", err) } timestamp := form.Get("timestamp") signature := []byte(form.Get("signature")) token := form.Get("token") mac := hmac.New(sha256.New, apikey) mac.Write([]byte(timestamp + token)) expected := []byte(hex.EncodeToString(mac.Sum(nil))) if !hmac.Equal(signature, expected) { return errors.New("invalid signature") } // Check uniqueness tokens := b.Bucket([]byte("tokens")) if p := tokens.Get([]byte(token)); p != nil { return errors.New("duplicate request token received") } return tokens.Put([]byte(token), []byte{}) }
func (self *TX) dig(keys [][]byte, create bool) (buckets []*bolt.Bucket, err error) { var bucket *bolt.Bucket if create { if bucket, err = self.tx.CreateBucketIfNotExists(keys[0]); err != nil { return } } else { if bucket = self.tx.Bucket(keys[0]); bucket == nil { err = ErrNotFound return } } buckets = append(buckets, bucket) for keys = keys[1:]; len(keys) > 0; keys = keys[1:] { if create { if bucket, err = bucket.CreateBucketIfNotExists(keys[0]); err != nil { return } } else { if bucket = bucket.Bucket(keys[0]); bucket == nil { err = ErrNotFound return } } buckets = append(buckets, bucket) } return }
func (store *BoltStore) multi_get(bucket *bolt.Bucket, suffix string, nested bool) ([]interface{}, error) { results := []interface{}{} if bucket != nil { c := bucket.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { if suffix == "" { if v != nil { vv, err := store.coder.Decode(v) if err != nil { return nil, err } results = append(results, vv) continue } if nested { bucket_nested := bucket.Bucket(k) if bucket_nested != nil { nests, err := store.multi_get(bucket_nested, suffix, nested) if err != nil { return results, err } results = append(results, nests...) } } } else { if strings.HasSuffix(string(k), suffix) { vv, err := store.get(bucket, string(k), true) if err != nil { return results, err } results = append(results, vv) continue } else { if nested && v == nil { bucket_nested := bucket.Bucket(k) if bucket_nested != nil { nests, err := store.multi_get(bucket_nested, suffix, nested) if err != nil { return results, err } results = append(results, nests...) } } } } } } return results, nil }
func bucketFor(key Key, b *bolt.Bucket) (*bolt.Bucket, []byte, error) { var err error elems, last := key.B() for _, e := range elems { // CreateBucketIfNotExists requires a writeable transaction. if new := b.Bucket(e); new != nil { b = new continue } if b, err = b.CreateBucket(e); err != nil { return b, last, err } } return b, last, nil }
func walkBucket(parent *bolt.Bucket, k []byte, v []byte, w io.Writer) error { if _, err := fmt.Fprintf(w, "%d:%x=%x\n", parent.Sequence(), k, v); err != nil { return err } // not a bucket, exit. if v != nil { return nil } return parent.ForEach(func(k, v []byte) error { if v == nil { return walkBucket(parent.Bucket(k), k, nil, w) } return walkBucket(parent, k, v, w) }) }
// GetBucket returns the given bucket below the current node. func (n *node) GetBucket(tx *bolt.Tx, children ...string) *bolt.Bucket { var b *bolt.Bucket bucketNames := append(n.rootBucket, children...) for _, bucketName := range bucketNames { if b != nil { if b = b.Bucket([]byte(bucketName)); b == nil { return nil } } else { if b = tx.Bucket([]byte(bucketName)); b == nil { return nil } } } return b }
// NewUniqueIndex loads a UniqueIndex func NewUniqueIndex(parent *bolt.Bucket, indexName []byte) (*UniqueIndex, error) { var err error b := parent.Bucket(indexName) if b == nil { if !parent.Writable() { return nil, ErrNotFound } b, err = parent.CreateBucket(indexName) if err != nil { return nil, err } } return &UniqueIndex{ IndexBucket: b, Parent: parent, }, nil }
func readBucket(b *bolt.Bucket) (*BoltBucket, error) { bb := new(BoltBucket) b.ForEach(func(k, v []byte) error { if v == nil { tb, err := readBucket(b.Bucket(k)) tb.parent = bb if err == nil { tb.name = string(k) bb.buckets = append(bb.buckets, *tb) } } else { tp := BoltPair{key: string(k), val: string(v)} tp.parent = bb bb.pairs = append(bb.pairs, tp) } return nil }) return bb, nil }
func (cmd *CompactCommand) walkBucket(b *bolt.Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error { // Execute callback. if err := fn(keypath, k, v, seq); err != nil { return err } // If this is not a bucket then stop. if v != nil { return nil } // Iterate over each child key/value. keypath = append(keypath, k) return b.ForEach(func(k, v []byte) error { if v == nil { bkt := b.Bucket(k) return cmd.walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn) } return cmd.walkBucket(b, keypath, k, v, b.Sequence(), fn) }) }
func (b *Bucket) fill(bucket *bolt.Bucket) { //fmt.Println("Fill bucket ", bucket.Root()) //fill subbuckets // bucket.Tx().ForEach(func(name []byte, buck *bolt.Bucket) error { // fmt.Println("subbucket ", string(name)) // subbuck := Bucket{ // Name: string(name), // } // fmt.Println("start fill ", string(name)) // subbuck.fill(buck) // b.Subbuckets = append(b.Subbuckets, subbuck) // return nil // }) //fill entries bucket.ForEach(func(k, v []byte) error { if len(v) == 0 { //subbucket sb := bucket.Bucket(k) if sb == nil { b.Entries = append(b.Entries, Entry{string(k), string(v)}) return nil } subbuck := Bucket{ Name: string(k), Subbuckets: []Bucket{}, Entries: []Entry{}, } subbuck.fill(sb) b.Subbuckets = append(b.Subbuckets, subbuck) } else { b.Entries = append(b.Entries, decodeEntry(k, v)) } return nil }) }
func getBucketContent(b *bolt.Bucket) (content map[string]string, nested []string) { content = map[string]string{} nested = []string{} b.ForEach(func(k, v []byte) error { if len(v) != 0 { var out bytes.Buffer json.Indent(&out, v, "", "\t") content[string(k)] = out.String() return nil } // may be k is a bucket nb := b.Bucket(k) if nb != nil { nested = append(nested, string(k)) } else { content[string(k)] = "" } return nil }) return content, nested }
// fetchNodeChannels retrieves all active channels from the target // nodeChanBucket. This function is typically used to fetch all the active // channels related to a particualr node. func (d *DB) fetchNodeChannels(openChanBucket, nodeChanBucket *bolt.Bucket) ([]*OpenChannel, error) { var channels []*OpenChannel // Once we have the node's channel bucket, iterate through each // item in the inner chan ID bucket. This bucket acts as an // index for all channels we currently have open with this node. nodeChanIDBucket := nodeChanBucket.Bucket(chanIDBucket[:]) if nodeChanIDBucket == nil { return nil, nil } err := nodeChanIDBucket.ForEach(func(k, v []byte) error { if k == nil { return nil } outBytes := bytes.NewReader(k) chanID := &wire.OutPoint{} if err := readOutpoint(outBytes, chanID); err != nil { return err } oChannel, err := fetchOpenChannel(openChanBucket, nodeChanBucket, chanID) if err != nil { return err } oChannel.Db = d channels = append(channels, oChannel) return nil }) if err != nil { return nil, err } return channels, nil }
// Process verifies the signature and uniqueness of the delivery identifier. func (GithubValidator) Process(h Hook, r Request, b *bolt.Bucket) error { // Check HMAC secret := b.Get([]byte(fmt.Sprintf("%s-secret", h.ID))) if secret == nil { return errors.New("github validator not initialized") } signature := []byte(r.Headers["X-Hub-Signature"]) mac := hmac.New(sha1.New, secret) mac.Write(bytes.TrimSpace(r.Body)) expected := append([]byte("sha1="), hex.EncodeToString(mac.Sum(nil))...) if !hmac.Equal(signature, expected) { return errors.New("invalid signature") } // Check uniqueness id := []byte(r.Headers["X-Github-Delivery"]) deliveries := b.Bucket([]byte("deliveries")) if did := deliveries.Get([]byte(id)); did != nil { return errors.New("duplicate delivery") } return deliveries.Put([]byte(id), []byte{}) }
// Process makes sure incoming requests do not exceed the configured rate // limit. func (RateLimitFilter) Process(h Hook, r Request, b *bolt.Bucket) error { amount, _ := strconv.Atoi(string(b.Get([]byte(fmt.Sprintf("%s-amount", h.ID))))) interval, _ := strconv.Atoi(string(b.Get([]byte(fmt.Sprintf("%s-interval", h.ID))))) if amount <= 0 || interval <= 0 { return errors.New("rate limit filter not initialized") } b = b.Bucket([]byte("requests")) // store current timestamp now := time.Now() k := []byte(fmt.Sprintf("%d", now.UnixNano())) if err := b.Put(k, nil); err != nil { return err } // count requests c := b.Cursor() from := []byte(fmt.Sprintf("%d", now.Add(time.Duration(-interval)*time.Second).UnixNano())) var count int for k, _ := c.Seek(from); k != nil; k, _ = c.Next() { count++ } if count > amount { return fmt.Errorf("rate limit exceeded (limit=%d count=%d)", amount, count) } // cleanup old entries for k, _ := c.First(); k != nil && bytes.Compare(k, from) <= 0; k, _ = c.Next() { if err := b.Delete(k); err != nil { return err } } return nil }
func exportBucket(b *bolt.Bucket) (*rawMessage, error) { // Encode individual key/value pairs into raw messages. var children = make([]*rawMessage, 0) err := b.ForEach(func(k, v []byte) error { var err error // If there is no value then it is a bucket. if v == nil { child, err := exportBucket(b.Bucket(k)) if err != nil { return fmt.Errorf("bucket: %s", err) } child.Key = k children = append(children, child) return nil } // Otherwise it's a regular key. var child = &rawMessage{Key: k} if child.Value, err = json.Marshal(v); err != nil { return fmt.Errorf("value: %s", err) } children = append(children, child) return nil }) if err != nil { return nil, err } // Encode bucket into a raw message. var root = rawMessage{Type: "bucket"} if root.Value, err = json.Marshal(children); err != nil { return nil, fmt.Errorf("children: %s", err) } return &root, nil }
func newMeta(b *bolt.Bucket, n Node) (*meta, error) { m := b.Bucket([]byte(metadataBucket)) if m != nil { name := m.Get([]byte(metaCodec)) if string(name) != n.Codec().Name() { return nil, ErrDifferentCodec } return &meta{ node: n, bucket: m, }, nil } m, err := b.CreateBucket([]byte(metadataBucket)) if err != nil { return nil, err } m.Put([]byte(metaCodec), []byte(n.Codec().Name())) return &meta{ node: n, bucket: m, }, nil }
// NewListIndex loads a ListIndex func NewListIndex(parent *bolt.Bucket, indexName []byte) (*ListIndex, error) { var err error b := parent.Bucket(indexName) if b == nil { if !parent.Writable() { return nil, ErrNotFound } b, err = parent.CreateBucket(indexName) if err != nil { return nil, err } } ids, err := NewUniqueIndex(b, []byte("storm__ids")) if err != nil { return nil, err } return &ListIndex{ IndexBucket: b, Parent: parent, IDs: ids, }, nil }
func openTabAll(bktBook *bolt.Bucket, tabid string) (*bolt.Bucket, *bolt.Bucket) { bktTab := bktBook.Bucket(bs("tab_" + tabid)) bktNotes := bktTab.Bucket(bs("notes")) bktPrevids := bktTab.Bucket(bs("previds")) return bktNotes, bktPrevids }
func openPrevids(bktTab *bolt.Bucket) *bolt.Bucket { return bktTab.Bucket(bs("previds")) }
func openNotes(bktTab *bolt.Bucket) *bolt.Bucket { return bktTab.Bucket(bs("notes")) }
func openTab(bktBook *bolt.Bucket, tabid string) *bolt.Bucket { return bktBook.Bucket(bs("tab_" + tabid)) }
func openTabs(bktBook *bolt.Bucket) *bolt.Bucket { return bktBook.Bucket(bs("tabs")) }