func getEncodedObject(bucket *bolt.Bucket, key string, value interface{}) error {
data := bucket.Get([]byte(key))
if data == nil {
return ObjectNotFoundErr
}
return json.Unmarshal(data, &value)
}
// targetAdjustmentBase returns the magnitude that the target should be
// adjusted by before a clamp is applied.
func (cs *ConsensusSet) targetAdjustmentBase(blockMap *bolt.Bucket, pb *processedBlock) *big.Rat {
// Grab the block that was generated 'TargetWindow' blocks prior to the
// parent. If there are not 'TargetWindow' blocks yet, stop at the genesis
// block.
var windowSize types.BlockHeight
parent := pb.Block.ParentID
current := pb.Block.ID()
for windowSize = 0; windowSize < types.TargetWindow && parent != (types.BlockID{}); windowSize++ {
current = parent
copy(parent[:], blockMap.Get(parent[:])[:32])
}
timestamp := types.Timestamp(encoding.DecUint64(blockMap.Get(current[:])[40:48]))
// The target of a child is determined by the amount of time that has
// passed between the generation of its immediate parent and its
// TargetWindow'th parent. The expected amount of seconds to have passed is
// TargetWindow*BlockFrequency. The target is adjusted in proportion to how
// time has passed vs. the expected amount of time to have passed.
//
// The target is converted to a big.Rat to provide infinite precision
// during the calculation. The big.Rat is just the int representation of a
// target.
timePassed := pb.Block.Timestamp - timestamp
expectedTimePassed := types.BlockFrequency * windowSize
return big.NewRat(int64(timePassed), int64(expectedTimePassed))
}
// Params returns the currently stored configuration parameters for hook h
// from bucket b.
func (RateLimitFilter) Params(h Hook, b *bolt.Bucket) map[string]string {
m := make(map[string]string)
for _, k := range []string{"amount", "interval"} {
m[k] = string(b.Get([]byte(fmt.Sprintf("%s-%s", h.ID, k))))
}
return m
}
func addCityToIndex(
bucket *bolt.Bucket, id string, name string, locale string, population uint32,
) error {
var err error
var cityName *ds.CityName
if locale == "" {
locale = "en"
}
cityNameKey := []byte(ds.PrepareCityNameKey(name))
if conflict := bucket.Get(cityNameKey); conflict != nil {
cityName, err = ds.CityNameFromString(string(cityNameKey), string(conflict))
if strconv.Itoa(cityName.CityId) != id {
cityNameKey = []byte(string(cityNameKey) + "|" + id)
}
}
err = bucket.Put(
cityNameKey, []byte(
name+"\t"+id+"\t"+locale+"\t"+strconv.Itoa(int(population)),
),
)
return err
}
// Params returns the currently stored configuration parameters for hook h
// from bucket b.
func (ForwardRequestAction) Params(h Hook, b *bolt.Bucket) map[string]string {
m := make(map[string]string)
for _, k := range []string{"url"} {
m[k] = string(b.Get([]byte(fmt.Sprintf("%s-%s", h.ID, k))))
}
return m
}
// earliestChildTimestamp returns the earliest timestamp that a child node
// can have while still being valid. See section 'Timestamp Rules' in
// Consensus.md.
//
// To boost performance, earliestChildTimestamp is passed a bucket that it can
// use from inside of a boltdb transaction.
func earliestChildTimestamp(blockMap *bolt.Bucket, pb *processedBlock) types.Timestamp {
// Get the previous MedianTimestampWindow timestamps.
windowTimes := make(types.TimestampSlice, types.MedianTimestampWindow)
windowTimes[0] = pb.Block.Timestamp
parent := pb.Parent
for i := uint64(1); i < types.MedianTimestampWindow; i++ {
// If the genesis block is 'parent', use the genesis block timestamp
// for all remaining times.
if parent == (types.BlockID{}) {
windowTimes[i] = windowTimes[i-1]
continue
}
// Get the next parent's bytes. Because the ordering is specific, the
// parent does not need to be decoded entirely to get the desired
// information. This provides a performance boost. The id of the next
// parent lies at the first 32 bytes, and the timestamp of the block
// lies at bytes 40-48.
parentBytes := blockMap.Get(parent[:])
copy(parent[:], parentBytes[:32])
windowTimes[i] = types.Timestamp(encoding.DecUint64(parentBytes[40:48]))
}
sort.Sort(windowTimes)
// Return the median of the sorted timestamps.
return windowTimes[len(windowTimes)/2]
}
// Params returns the currently stored configuration parameters for hook h
// from bucket b.
func (ExecuteAction) Params(h Hook, b *bolt.Bucket) map[string]string {
m := make(map[string]string)
for _, k := range []string{"command"} {
m[k] = string(b.Get([]byte(fmt.Sprintf("%s-%s", h.ID, k))))
}
return m
}
// Params returns the currently stored configuration parameters for hook h
// from bucket b.
func (EmailAction) Params(h Hook, b *bolt.Bucket) map[string]string {
m := make(map[string]string)
for _, k := range []string{"token", "domain", "address", "subject", "template"} {
m[k] = string(b.Get([]byte(fmt.Sprintf("%s-%s", h.ID, k))))
}
return m
}
func getGrantScopes(bucket *bolt.Bucket, grant string) map[string]bool {
scopesByte := bucket.Get([]byte(grant))
if scopesByte == nil {
return nil
}
return database.StringToSet(string(scopesByte))
}
func (store *BoltStore) size(bucket *bolt.Bucket, key string, size *AtomicInt64) int64 {
var statics *AtomicInt64 = size
if statics == nil {
statics = NewAtomicInt64(0)
}
if key != "" {
b := bucket.Get([]byte(key))
if b != nil {
return statics.Caculate(int64(len(key) + len(b)))
}
bucket = bucket.Bucket([]byte(key))
}
if bucket != nil {
c := bucket.Cursor()
// to do staticsing
for k, v := c.First(); k != nil; k, v = c.Next() {
if v != nil {
statics.Caculate(int64(len(k) + len(v)))
} else {
store.size(bucket, string(k), statics)
}
}
statics.Caculate(int64(len(key)))
return statics.Value()
}
//! null
return statics.Value()
}
func (store *BoltStore) get(bucket *bolt.Bucket, key string, nested bool) (interface{}, error) {
if key != "" {
b := bucket.Get([]byte(key))
if b != nil {
return store.coder.Decode(b)
}
bucket = bucket.Bucket([]byte(key))
}
if bucket != nil {
m := make(map[string]interface{})
c := bucket.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v != nil {
vv, err := store.coder.Decode(v)
if err != nil {
return nil, err
}
m[string(k)] = vv
}
if v == nil && nested == true {
vv, err := store.get(bucket, string(k), nested)
if err != nil {
return nil, err
}
m[string(k)] = vv
}
}
return m, nil
}
return nil, fmt.Errorf("bucket not exist")
}
// Store a word,word -> word sequence
func storeWords(bucket *bolt.Bucket, word1, word2, word3 string) error {
key := []byte(word1 + " " + word2)
// Get value from bucket and decode it
rawValue := bucket.Get(key)
var value []string
if rawValue == nil {
value = make([]string, 0, 1)
} else {
if err := json.Unmarshal(rawValue, &value); err != nil {
log.Printf("Cannot decode raw value for key '%v': %v; starting new empty key; old value is: %v", string(key), string(rawValue))
value = make([]string, 0, 1)
}
}
// Add new word to value
value = append(value, word3)
// Encode value and store it in bucket
rawValue, err := json.Marshal(value)
if err != nil {
return err
}
if err := bucket.Put(key, rawValue); err != nil {
return err
}
// All done
return nil
}
func (this *DB) getData(bucket *bolt.Bucket) (map[string]int, ipQueue, error) {
var addrQ ipQueue
heapRaw := bucket.Get(boltAddrHeapKey)
if heapRaw == nil {
addrQ = ipQueue(make([]*ipEntry, 0, 1))
} else {
dec := gob.NewDecoder(bytes.NewReader(heapRaw))
if err := dec.Decode(&addrQ); err != nil {
return nil, nil, err
}
}
var addrMap map[string]int
mapRaw := bucket.Get(boltAddrMapKey)
if mapRaw == nil {
addrMap = make(map[string]int)
} else {
dec := gob.NewDecoder(bytes.NewReader(mapRaw))
if err := dec.Decode(&addrMap); err != nil {
return nil, nil, err
}
}
return addrMap, addrQ, nil
}
// Params returns the currently stored configuration parameters for hook h
// from bucket b.
func (GithubValidator) Params(h Hook, b *bolt.Bucket) map[string]string {
m := make(map[string]string)
for _, k := range []string{"secret"} {
m[k] = string(b.Get([]byte(fmt.Sprintf("%s-%s", h.ID, k))))
}
return m
}
func fetchChanCommitKeys(nodeChanBucket *bolt.Bucket, channel *OpenChannel) error {
// Construct the key which stores the commitment keys: ckk || channelID.
// TODO(roasbeef): factor into func
var bc bytes.Buffer
if err := writeOutpoint(&bc, channel.ChanID); err != nil {
return err
}
commitKey := make([]byte, len(commitKeys)+bc.Len())
copy(commitKey[:3], commitKeys)
copy(commitKey[3:], bc.Bytes())
var err error
keyBytes := nodeChanBucket.Get(commitKey)
channel.TheirCommitKey, err = btcec.ParsePubKey(keyBytes[:33], btcec.S256())
if err != nil {
return err
}
channel.OurCommitKey, err = btcec.ParsePubKey(keyBytes[33:], btcec.S256())
if err != nil {
return err
}
return nil
}
func GetLink(links *bolt.Bucket, stats *het.CountStats, url *url.URL) (het.Link, error) {
url.Fragment = ""
lbytes := links.Get([]byte(url.String()))
link := het.Link{}
if lbytes != nil {
// link already exists, return early
json.Unmarshal(lbytes, &link)
// follow redirects in the links bucket
if link.Redirect {
return GetLink(links, stats, &link.URL)
}
return link, nil
}
resp, err := http.Get(url.String())
if err != nil {
return link, err
}
defer resp.Body.Close()
finalURL := resp.Request.URL
finalURL.Fragment = ""
link = het.Link{
URL: *finalURL,
StatusCode: resp.StatusCode,
ContentType: resp.Header.Get("Content-Type"),
LastModified: strings.Trim(resp.Header.Get("Last-Modified"), " \t\n"),
}
lbytes, err = json.Marshal(&link)
if err != nil {
log.Fatal(err)
}
links.Put([]byte(finalURL.String()), lbytes)
stats.LinkCount++
// redirect link
if finalURL.String() != url.String() {
lrbytes, err := json.Marshal(&het.Link{
URL: *finalURL,
Redirect: true,
})
if err != nil {
log.Fatal(err)
}
links.Put([]byte(url.String()), lrbytes)
stats.LinkCount++
}
return link, nil
}
func executeSelect(stmt *boltq.SelectStatement, db *bolt.DB) error {
return db.View(func(tx *bolt.Tx) error {
var bucket *bolt.Bucket
for _, name := range stmt.BucketPath {
log.Debugln("navigating to bucket", name)
bucket = tx.Bucket([]byte(name))
if bucket == nil {
return fmt.Errorf("cannot find bucket %s", name)
}
}
if containsAsterisk(stmt.Fields) {
log.Debugln("interating keys")
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
emitKeypair(k, v)
}
} else {
for _, k := range stmt.Fields {
keyBytes := []byte(k)
v := bucket.Get(keyBytes)
emitKeypair(keyBytes, v)
}
}
return nil
})
}
// Process verifies the signature and uniqueness of the random roken.
func (MailgunValidator) Process(h Hook, r Request, b *bolt.Bucket) error {
// Check HMAC
apikey := b.Get([]byte(fmt.Sprintf("%s-apikey", h.ID)))
if apikey == nil {
return errors.New("mailgun validator not initialized")
}
if r.Headers["Content-Type"] != "application/x-www-form-urlencoded" {
return fmt.Errorf("unexpected Content-Type: %q", r.Headers["Content-Type"])
}
form, err := url.ParseQuery(string(r.Body))
if err != nil {
return fmt.Errorf("error parsing request body: %s", err)
}
timestamp := form.Get("timestamp")
signature := []byte(form.Get("signature"))
token := form.Get("token")
mac := hmac.New(sha256.New, apikey)
mac.Write([]byte(timestamp + token))
expected := []byte(hex.EncodeToString(mac.Sum(nil)))
if !hmac.Equal(signature, expected) {
return errors.New("invalid signature")
}
// Check uniqueness
tokens := b.Bucket([]byte("tokens"))
if p := tokens.Get([]byte(token)); p != nil {
return errors.New("duplicate request token received")
}
return tokens.Put([]byte(token), []byte{})
}
// get gob decodes the value for key in the given bucket into val. It reports
// if a value with that key exists, and any error in retrieving or decoding the
// value.
func get(b *bolt.Bucket, key []byte, val interface{}) (bool, error) {
v := b.Get(key)
if v == nil {
return false, nil
}
d := gob.NewDecoder(bytes.NewReader(v))
return true, d.Decode(val)
}
func pluginFrom(store *bolt.Bucket, name string) (*pluginT, error) {
data := store.Get([]byte(name))
if data == nil {
return nil, fmt.Errorf("Plugin '%s' was not found", name)
}
return decodePlugin(data, name)
}
func queryString(bucket *bolt.Bucket, key string) string {
value := bucket.Get([]byte(key))
if value == nil {
return ""
} else {
return string(value)
}
}
// commitSiacoinOutputDiff applies or reverts a SiacoinOutputDiff from within
// a database transaction.
func (cs *ConsensusSet) commitBucketSiacoinOutputDiff(scoBucket *bolt.Bucket, scod modules.SiacoinOutputDiff, dir modules.DiffDirection) error {
if build.DEBUG && (scoBucket.Get(scod.ID[:]) == nil) != (scod.Direction == dir) {
panic(errRepeatInsert)
}
if scod.Direction == dir {
return scoBucket.Put(scod.ID[:], encoding.Marshal(scod.SiacoinOutput))
}
return scoBucket.Delete(scod.ID[:])
}
func fetchInvoice(invoiceNum []byte, invoices *bolt.Bucket) (*Invoice, error) {
invoiceBytes := invoices.Get(invoiceNum)
if invoiceBytes == nil {
return nil, ErrInvoiceNotFound
}
invoiceReader := bytes.NewReader(invoiceBytes)
return deserializeInvoice(invoiceReader)
}
func getItemsFromBucket(bucket *bolt.Bucket, tagName string) ([]int64, error) {
bts := bucket.Get([]byte(tagName))
sl, e := intarr.Decode(bts)
if e != nil {
return nil, e
}
return sl.Int64(), e
}
func addTranslationsToCountry(
bucket *bolt.Bucket, id int, translations []string,
) error {
key := strconv.Itoa(id)
val := bucket.Get([]byte(key))
return bucket.Put([]byte(key), []byte(
string(val)+";"+strings.Join(translations, ";"),
))
}
func bkPutNX(b *bolt.Bucket, k, v []byte) (ok bool, err error) {
if b.Get(k) != nil {
ok = false
return
}
if err = b.Put(k, v); err == nil {
ok = true
}
return
}
func LoadOneBoltTx(bucket *bolt.Bucket, key string, obj proto.Unmarshaler) error {
objBytes := bucket.Get([]byte(key))
if objBytes == nil {
return fmt.Errorf("Key not found: '%s'", key)
}
err := obj.Unmarshal(objBytes)
if err != nil {
return fmt.Errorf("Could not unmarshal key %s", key)
}
return nil
}
func nextDocID(b *bolt.Bucket) (uint64, error) {
v := b.Get([]byte("max_id"))
if v == nil {
return 0, nil
}
maxID, read := binary.Uvarint(v)
if read <= 0 {
return 0, fmt.Errorf("Error converting %#v to a uint64", v)
}
return maxID + 1, nil
}
func getEntryUnsafely(bucket *bolt.Bucket, blockHash []byte) (fileCacheEntry, bool) {
v := bucket.Get(blockHash)
if v == nil {
// not found, escape!
return fileCacheEntry{}, false
}
buf := bytes.NewBuffer(v)
dec := gob.NewDecoder(buf)
var entry fileCacheEntry
dec.Decode(&entry)
return entry, true
}
func (s *BoltStore) getFromBucket(b *bolt.Bucket, u *url.URL) (uu *crawler.URL, err error) {
us := u.String()
v := b.Get([]byte(us))
if v == nil {
return nil, errors.New("boltstore: item is not found")
}
w := &wrapper{}
err = s.codec.Unmarshal(v, w)
if err == nil {
uu, err = w.To(us)
}
return
}