// mustContainKeys checks that a bucket contains a given set of keys.
func mustContainKeys(b *bolt.Bucket, m map[string]string) {
found := make(map[string]string)
b.ForEach(func(k, _ []byte) error {
found[string(k)] = ""
return nil
})
// Check for keys found in bucket that shouldn't be there.
var keys []string
for k, _ := range found {
if _, ok := m[string(k)]; !ok {
keys = append(keys, k)
}
}
if len(keys) > 0 {
sort.Strings(keys)
panic(fmt.Sprintf("keys found(%d): %s", len(keys), strings.Join(keys, ",")))
}
// Check for keys not found in bucket that should be there.
for k, _ := range m {
if _, ok := found[string(k)]; !ok {
keys = append(keys, k)
}
}
if len(keys) > 0 {
sort.Strings(keys)
panic(fmt.Sprintf("keys not found(%d): %s", len(keys), strings.Join(keys, ",")))
}
}
func (rpcMethod *RPCMethod) listIDs(response *ResponseParameters) error {
//open a read transaction
rpcMethod.rpcServer.boltDB.View(func(tx *bolt.Tx) error {
var cursor *bolt.Cursor
cursor = tx.Cursor()
var bucket *bolt.Bucket
response.Result = make([]interface{}, 0, 10)
//traverse through all keys
for k, _ := cursor.First(); k != nil; k, _ = cursor.Next() {
bucket = tx.Bucket(k)
//traverse through all relation and value pairs
bucket.ForEach(func(relation, value []byte) error {
tuple := make([]string, 2)
rpcMethod.rpcServer.logger.Println(string(k), string(relation), string(value))
//make an array of 2 strings [key,relation]
tuple[0] = string(k)
tuple[1] = string(relation)
response.Result = append(response.Result, tuple)
return nil
})
}
return nil
})
response.Error = nil
return nil
}
/*
Called in Shutdown - keys are transferred to successor
*/
func (rpcServer *RPCServer) makeInsertsToSuccessor() {
//open a read transaction
rpcServer.boltDB.View(func(tx *bolt.Tx) error {
var cursor *bolt.Cursor
cursor = tx.Cursor()
var bucket *bolt.Bucket
//traverse through all keys
for k, _ := cursor.First(); k != nil; k, _ = cursor.Next() {
bucket = tx.Bucket(k)
//traverse through all relation and value pairs
bucket.ForEach(func(relation, value []byte) error {
//create paramter - successor
//add to array of interface
parameterArray := make([]interface{}, 3)
parameterArray[0] = string(k)
parameterArray[1] = string(relation)
parameterArray[2] = string(value)
//create json message
jsonMessage := rpcclient.RequestParameters{}
jsonMessage.Method = "Insert"
jsonMessage.Params = parameterArray
jsonBytes, err := json.Marshal(jsonMessage)
if err != nil {
rpcServer.logger.Println(err)
return err
}
rpcServer.logger.Println(string(jsonBytes))
clientServerInfo, err := rpcServer.chordNode.PrepareClientServerInfo(rpcServer.chordNode.FingerTable[1])
if err != nil {
rpcServer.logger.Println(err)
return nil
}
client := &rpcclient.RPCClient{}
err, _ = client.RpcCall(clientServerInfo, string(jsonBytes))
if err != nil {
rpcServer.logger.Println(err)
return nil
}
return nil
})
}
return nil
})
}
func walkBucket(parent *bolt.Bucket, k []byte, v []byte, w io.Writer) error {
if _, err := fmt.Fprintf(w, "%d:%x=%x\n", parent.Sequence(), k, v); err != nil {
return err
}
// not a bucket, exit.
if v != nil {
return nil
}
return parent.ForEach(func(k, v []byte) error {
if v == nil {
return walkBucket(parent.Bucket(k), k, nil, w)
}
return walkBucket(parent, k, v, w)
})
}
func instrumentsFromDB(b *bolt.Bucket) {
instruments = make([]Instrument, b.Stats().KeyN)
i := 0
b.ForEach(func(k, v []byte) error {
var inst Instrument
buf := bytes.NewBuffer(v)
dec := gob.NewDecoder(buf)
err := dec.Decode(&inst)
if err != nil {
return err
}
instruments[inst.ID] = inst
i++
return err
})
}
func listPlugins(store *bolt.Bucket) ([]*pluginT, error) {
plugins := []*pluginT{}
err := store.ForEach(func(key, data []byte) error {
plugin, err := decodePlugin(data, string(key))
if err != nil {
fmt.Printf("[ERROR] %s\n", err)
}
plugins = append(plugins, plugin)
return nil
})
if err != nil {
return nil, fmt.Errorf("Unable to fetch plugins from store - %s", err)
}
return plugins, nil
}
func (db *bwdb) sumAndShift(pullBkt, putBkt *bolt.Bucket, putKey []byte) error {
//pull all the values out of the pullBkt and sum them
s := db.newVar()
err := pullBkt.ForEach(func(k, v []byte) error {
sx := db.newVar()
if err := sx.Decode(v); err != nil {
return err
}
if err := s.Add(sx); err != nil {
return err
}
s.SetTS(sx.TS())
return pullBkt.Delete(k)
})
if err != nil {
return err
}
return db.updateVal(putBkt, putKey, s) //if for some reason the key already exists, add it
}
func readBucket(b *bolt.Bucket) (*BoltBucket, error) {
bb := new(BoltBucket)
b.ForEach(func(k, v []byte) error {
if v == nil {
tb, err := readBucket(b.Bucket(k))
tb.parent = bb
if err == nil {
tb.name = string(k)
bb.buckets = append(bb.buckets, *tb)
}
} else {
tp := BoltPair{key: string(k), val: string(v)}
tp.parent = bb
bb.pairs = append(bb.pairs, tp)
}
return nil
})
return bb, nil
}
func (cmd *CompactCommand) walkBucket(b *bolt.Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error {
// Execute callback.
if err := fn(keypath, k, v, seq); err != nil {
return err
}
// If this is not a bucket then stop.
if v != nil {
return nil
}
// Iterate over each child key/value.
keypath = append(keypath, k)
return b.ForEach(func(k, v []byte) error {
if v == nil {
bkt := b.Bucket(k)
return cmd.walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn)
}
return cmd.walkBucket(b, keypath, k, v, b.Sequence(), fn)
})
}
// Get 2 random start words
func getStart(bucket *bolt.Bucket) (string, string, error) {
var start []byte
var total int = 0
if err := bucket.ForEach(func(key, _ []byte) error {
total++
if rand.Intn(total) == 0 {
start = key
}
return nil
}); err != nil {
return "", "", err
}
var words = strings.Fields(string(start))
if len(words) != 2 {
return "", "", fmt.Errorf("Start '%v' does not contain exactly 2 words", string(start))
}
return words[0], words[1], nil
}
func (b *Bucket) fill(bucket *bolt.Bucket) {
//fmt.Println("Fill bucket ", bucket.Root())
//fill subbuckets
// bucket.Tx().ForEach(func(name []byte, buck *bolt.Bucket) error {
// fmt.Println("subbucket ", string(name))
// subbuck := Bucket{
// Name: string(name),
// }
// fmt.Println("start fill ", string(name))
// subbuck.fill(buck)
// b.Subbuckets = append(b.Subbuckets, subbuck)
// return nil
// })
//fill entries
bucket.ForEach(func(k, v []byte) error {
if len(v) == 0 { //subbucket
sb := bucket.Bucket(k)
if sb == nil {
b.Entries = append(b.Entries, Entry{string(k), string(v)})
return nil
}
subbuck := Bucket{
Name: string(k),
Subbuckets: []Bucket{},
Entries: []Entry{},
}
subbuck.fill(sb)
b.Subbuckets = append(b.Subbuckets, subbuck)
} else {
b.Entries = append(b.Entries, decodeEntry(k, v))
}
return nil
})
}
func getBucketContent(b *bolt.Bucket) (content map[string]string, nested []string) {
content = map[string]string{}
nested = []string{}
b.ForEach(func(k, v []byte) error {
if len(v) != 0 {
var out bytes.Buffer
json.Indent(&out, v, "", "\t")
content[string(k)] = out.String()
return nil
}
// may be k is a bucket
nb := b.Bucket(k)
if nb != nil {
nested = append(nested, string(k))
} else {
content[string(k)] = ""
}
return nil
})
return content, nested
}
func exportBucket(b *bolt.Bucket) (*rawMessage, error) {
// Encode individual key/value pairs into raw messages.
var children = make([]*rawMessage, 0)
err := b.ForEach(func(k, v []byte) error {
var err error
// If there is no value then it is a bucket.
if v == nil {
child, err := exportBucket(b.Bucket(k))
if err != nil {
return fmt.Errorf("bucket: %s", err)
}
child.Key = k
children = append(children, child)
return nil
}
// Otherwise it's a regular key.
var child = &rawMessage{Key: k}
if child.Value, err = json.Marshal(v); err != nil {
return fmt.Errorf("value: %s", err)
}
children = append(children, child)
return nil
})
if err != nil {
return nil, err
}
// Encode bucket into a raw message.
var root = rawMessage{Type: "bucket"}
if root.Value, err = json.Marshal(children); err != nil {
return nil, fmt.Errorf("children: %s", err)
}
return &root, nil
}
// GetRecordsForBucket use the bucket to look all the records
func GetRecordsForBucket(b *bolt.Bucket) (map[string]interface{}, error) {
m := make(map[string]interface{})
var vj Note
var kj interface{}
err := b.ForEach(func(k []byte, v []byte) error {
err := json.Unmarshal(v, &vj)
if err != nil {
log.Error(uuid, "report.getrecordsforbucket", err, "Unmarshalling.")
}
err = json.Unmarshal(k, &kj)
if err != nil {
log.Error(uuid, "report.getrecordsforbucket", err, "Unmarshalling.")
}
m[vj.ID] = vj
return err
})
return m, err
}