// AtomicDelete deletes a value at "key" if the key
// has not been modified in the meantime, throws an
// error if this is the case
func (b *BoltDB) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
var (
val []byte
db *bolt.DB
err error
)
if previous == nil {
return false, store.ErrPreviousNotSpecified
}
if db, err = b.getDBhandle(); err != nil {
return false, err
}
defer b.releaseDBhandle()
err = db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(b.boltBucket)
if bucket == nil {
return ErrBoltBucketNotFound
}
val = bucket.Get([]byte(key))
dbIndex := binary.LittleEndian.Uint64(val[:libkvmetadatalen])
if dbIndex != previous.LastIndex {
return store.ErrKeyModified
}
err := bucket.Delete([]byte(key))
return err
})
if err != nil {
return false, err
}
return true, err
}
func ShowStatus(db *bolt.DB, u string) error {
ur, err := url.Parse(u)
if err != nil {
return err
}
fmt.Printf("Host %s status:\n", ur.Host)
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(ur.Host))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var sr StatRecord
err := json.Unmarshal(v, &sr)
if err != nil {
fmt.Println(err)
c.Next()
}
fmt.Printf("%s => %s\n", k, sr)
}
fmt.Printf("\n\n")
return nil
})
return nil
}
func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
var t = time.Now()
for {
c := tx.Bucket(benchBucketName).Cursor()
var count int
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return errors.New("invalid value")
}
count++
}
if options.WriteMode == "seq" && count != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
// init initializes the database for this layer setup.
func (l *Layered) init(db *bolt.DB) ([]*layerVertex, error) {
layerVertices := make([]*layerVertex, len(l.Layers))
for i, layer := range l.Layers {
var parent *Layer
if i > 0 {
parent = l.Layers[i-1]
}
layerVertex, err := l.initLayer(db, layer, parent)
if err != nil {
return nil, err
}
layerVertices[i] = layerVertex
if parent != nil {
// We have a prior layer, so setup the edge pointer
err = db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltEdgesBucket)
return bucket.Put(
[]byte(layer.ID),
[]byte(parent.ID))
})
if err != nil {
return nil, err
}
}
}
return layerVertices, nil
}
func (l *Layered) deleteLayer(db *bolt.DB, layer *Layer, path string) error {
if err := os.RemoveAll(path); err != nil {
return err
}
return db.Update(func(tx *bolt.Tx) error {
// Delete the layer itself
bucket := tx.Bucket(boltLayersBucket)
key := []byte(layer.ID)
if err := bucket.Delete(key); err != nil {
return err
}
// Delete all the edges
bucket = tx.Bucket(boltEdgesBucket)
if err := bucket.Delete(key); err != nil {
return err
}
// Find any values
return bucket.ForEach(func(k, data []byte) error {
if string(data) == layer.ID {
return bucket.Delete(k)
}
return nil
})
})
}
func benchWriteNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
results.WriteOps = options.Iterations
for i := 0; i < options.Iterations; i += options.BatchSize {
err := db.Update(func(tx *bolt.Tx) error {
top, _ := tx.CreateBucketIfNotExists(benchBucketName)
var name = make([]byte, options.KeySize)
binary.BigEndian.PutUint32(name, keySource())
b, _ := top.CreateBucketIfNotExists(name)
for j := 0; j < options.BatchSize; j++ {
var key = make([]byte, options.KeySize)
var value = make([]byte, options.ValueSize)
binary.BigEndian.PutUint32(key, keySource())
if err := b.Put(key, value); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
}
return nil
}
func addUser(db *bolt.DB, email string, password string) bool {
check := []byte("")
hashedPass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
fmt.Println(err)
}
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("UsersBucket"))
check = b.Get([]byte(email))
return nil
})
if check == nil {
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("UsersBucket"))
err := b.Put([]byte(email), []byte(hashedPass))
return err
})
if err != nil {
fmt.Println(err)
}
return true
} else {
return false
}
}
// QueryNodeByTaxID querys Node by taxid
func QueryNodeByTaxID(db *bolt.DB, bucket string, taxids []string) ([]nodes.Node, error) {
for _, taxid := range taxids {
if !reDigitals.MatchString(taxid) {
return []nodes.Node{}, fmt.Errorf("non-digital taxid given: %s", taxid)
}
}
nods := make([]nodes.Node, len(taxids))
if len(taxids) == 0 {
return nods, nil
}
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("database not exists: %s", bucket)
}
for i, taxid := range taxids {
s := string(b.Get([]byte(taxid)))
if s == "" {
nods[i] = nodes.Node{}
continue
}
node, err := nodes.NodeFromJSON(s)
if err != nil {
return errors.New("failed to parse node record from database")
}
nods[i] = node
}
return nil
})
return nods, err
}
func (s *server) dump(db *bolt.DB, changes map[uint64]bool) {
for k := range changes {
// marshal
var rs *RankSet
s.lock_read(func() {
rs = s.ranks[k]
})
if rs == nil {
log.Warning("empty rankset:", k)
continue
}
// serialization and save
bin, err := rs.Marshal()
if err != nil {
log.Critical("cannot marshal:", err)
os.Exit(-1)
}
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(BOLTDB_BUCKET))
err := b.Put([]byte(fmt.Sprint(k)), bin)
return err
})
}
}
// *whirrrrrrrrrrr*
func probe(db *bolt.DB, api *tbotapi.TelegramBotAPI, closed chan struct{}, wg *sync.WaitGroup) {
dbBucketName := "alphazero"
dbBucketStats := fmt.Sprintf("%s.stats", dbBucketName)
for {
select {
case <-closed:
wg.Done()
return
case val := <-api.Updates:
typ := val.Message.Type()
if typ != model.TextType {
fmt.Fprintf(os.Stderr, "Not handling type: %s\n", typ)
continue
}
response, md := process(db, val)
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(dbBucketStats))
id := []byte("messages.count.received")
var val []byte
if val = b.Get(id); val == nil {
val = []byte("0")
}
c, _ := strconv.Atoi(string(val))
c++
return b.Put(id, []byte(strconv.Itoa(c)))
})
// unsafeval := blackfriday.MarkdownCommon([]byte(*val.Message.Text))
// unsafe := blackfriday.MarkdownCommon([]byte(response))
// html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
sender := val.Message.Chat
recipient := model.NewRecipientFromChat(val.Message.Chat)
om := model.NewOutgoingMessage(
model.NewChatRecipient(*recipient.ChatID),
response).SetMarkdown(md)
var msg *model.MessageResponse
var err error
msg, err = api.SendMessageExtended(om)
if err != nil {
msg, err = api.SendMessageExtended(om.SetMarkdown(false))
if err != nil {
fmt.Fprintf(os.Stderr, "Err: %s\n", err)
continue
}
}
fmt.Printf(
"MessageID: %04d, ChatID: %s, Text: %s, IsGroupChat:%t\n",
msg.Message.ID, sender.String(), response, msg.Message.Chat.IsGroupChat())
case val := <-api.Errors:
fmt.Printf("Err: %s\n", val)
}
}
}
// LoadAllNodes loads all nodes into memory
func LoadAllNodes(db *bolt.DB, bucket string) (map[string]nodes.Node, error) {
nods := make(map[string]nodes.Node)
ch := make(chan string, runtime.NumCPU())
chDone := make(chan int)
go func() {
for s := range ch {
node, err := nodes.NodeFromJSON(s)
checkError(err)
nods[node.TaxID] = node
}
chDone <- 1
}()
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("database not exists: %s", bucket)
}
b.ForEach(func(k, v []byte) error {
ch <- string(v)
return nil
})
return nil
})
close(ch)
<-chDone
return nods, err
}
func printDBDebug(db *bolt.DB) {
dbBucketName := "alphazero"
dbBucketLogs := fmt.Sprintf("%s.logs", dbBucketName)
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(dbBucketLogs))
c := b.Cursor()
i := 10
for k, v := c.Last(); k != nil; k, v = c.Prev() {
log := LogEntry{}
json.Unmarshal(v, &log)
fmt.Printf("- %s\n", log.String())
i--
if i == 0 {
break
}
}
return nil
})
dbBucketStats := fmt.Sprintf("%s.stats", dbBucketName)
db.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(dbBucketStats)).Cursor()
prefix := []byte("stats.")
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
fmt.Printf("key=%s, value=%s\n", k, v)
}
return nil
})
}
func logAction(db *bolt.DB, action string) {
dbBucketName := "alphazero"
dbBucketLogs := fmt.Sprintf("%s.logs", dbBucketName)
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(dbBucketLogs))
id, _ := b.NextSequence()
l := LogEntry{
ID: id,
When: strconv.FormatInt(time.Now().Unix(), 10),
Action: action,
UUID: uuid.NewV4().String(),
}
buf, err := json.Marshal(l)
if err != nil {
return err
}
err = b.Put(itob(l.ID), buf)
if err != nil {
return err
}
return nil
})
}
func (c *Context) SerializeIndex(index *Index) (err error) {
var data []byte
var db *bolt.DB
if data, err = json.Marshal(index); err != nil {
return
}
db, err = c.OpenDB()
if err != nil {
return err
}
defer db.Close()
return db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(byteify(IndicesKey))
if err != nil {
return err
}
if bucket == nil {
return ErrNoSuchDbBucket
}
return bucket.Put(byteify(index.FileId), data)
})
}
// newBroadcastsStore creates a new BroadcastAddresses object after doing the
// necessary initialization.
func newBroadcastsStore(db *bolt.DB, username string) (*BroadcastAddresses, error) {
b := &BroadcastAddresses{
db: db,
username: []byte(username),
addrs: make([]bmutil.Address, 0),
}
err := db.Update(func(tx *bolt.Tx) error {
userbucket, err := tx.CreateBucketIfNotExists([]byte(username))
if err != nil {
return err
}
bucket, err := userbucket.CreateBucketIfNotExists(broadcastAddressesBucket)
if err != nil {
return err
}
return bucket.ForEach(func(k, _ []byte) error {
addr, err := bmutil.DecodeAddress(string(k))
if err != nil {
return err
}
b.addrs = append(b.addrs, *addr)
return nil
})
})
if err != nil {
return nil, err
}
return b, nil
}
func InsertData(db *bolt.DB, artist string, track string, cover string) {
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(DATABASE_BUCKET)
err := b.Put([]byte(artist+" - "+track), []byte(cover))
return err
})
}
func NewMockAllocator(db *bolt.DB) *MockAllocator {
d := &MockAllocator{}
d.clustermap = make(map[string]sort.StringSlice)
var clusters []string
err := db.View(func(tx *bolt.Tx) error {
var err error
clusters, err = ClusterList(tx)
if err != nil {
return err
}
for _, cluster := range clusters {
err := d.addDevicesFromDb(tx, cluster)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return nil
}
return d
}
//Put the key, value pair. index number metadata is prepended to the value
func (b *BoltDB) Put(key string, value []byte, opts *store.WriteOptions) error {
var (
dbIndex uint64
db *bolt.DB
err error
)
b.Lock()
defer b.Unlock()
dbval := make([]byte, libkvmetadatalen)
if db, err = b.getDBhandle(); err != nil {
return err
}
defer b.releaseDBhandle()
err = db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(b.boltBucket)
if err != nil {
return err
}
dbIndex = atomic.AddUint64(&b.dbIndex, 1)
binary.LittleEndian.PutUint64(dbval, dbIndex)
dbval = append(dbval, value...)
err = bucket.Put([]byte(key), dbval)
if err != nil {
return err
}
return nil
})
return err
}
func writePageAttribute(PageDB *bolt.DB, AttrName, AttrValue string) error {
UpdateFuncErr := PageDB.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("PageAttributes"))
return bucket.Put([]byte(AttrName), []byte(AttrValue))
})
return UpdateFuncErr
}
// Exists checks if the key exists inside the store
func (b *BoltDB) Exists(key string) (bool, error) {
var (
val []byte
db *bolt.DB
err error
)
b.Lock()
defer b.Unlock()
if db, err = b.getDBhandle(); err != nil {
return false, err
}
defer b.releaseDBhandle()
err = db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(b.boltBucket)
if bucket == nil {
return store.ErrKeyNotFound
}
val = bucket.Get([]byte(key))
return nil
})
if len(val) == 0 {
return false, err
}
return true, err
}
func DoRead(db *bolt.DB) error {
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
log.Println("b is %s", b)
//if err != nil {
// return err
//}
v := b.Get([]byte("answer"))
//log.Printf("v=%s", v)
//if err != nil {
// return err
//}
//ctrl, err := JobControlFromBytes(v)
ctrl := &JobControl{}
err := ctrl.FromBytes(v)
if err != nil {
log.Println("error", err)
return err
}
log.Printf("the value is %+v", ctrl)
return nil
})
return err
}
// DeleteTree deletes a range of keys with a given prefix
func (b *BoltDB) DeleteTree(keyPrefix string) error {
var (
db *bolt.DB
err error
)
b.Lock()
defer b.Unlock()
if db, err = b.getDBhandle(); err != nil {
return err
}
defer b.releaseDBhandle()
err = db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(b.boltBucket)
if bucket == nil {
return store.ErrKeyNotFound
}
cursor := bucket.Cursor()
prefix := []byte(keyPrefix)
for key, _ := cursor.Seek(prefix); bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
_ = bucket.Delete([]byte(key))
}
return nil
})
return err
}
func (l *Layered) updateLayer(db *bolt.DB, layer *Layer, f func(*layerVertex)) error {
return db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(boltLayersBucket)
key := []byte(layer.ID)
data := bucket.Get(key)
if len(data) == 0 {
// This should never happen through this struct
panic(fmt.Errorf("layer %s not found", layer.ID))
}
// Read the vertex, call the function to modify it
var v layerVertex
if err := l.structRead(&v, data); err != nil {
return err
}
f(&v)
// Save the resulting layer data
data, err := l.structData(&v)
if err != nil {
return err
}
return bucket.Put(key, data)
})
}
func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
results.WriteOps = options.Iterations
for i := 0; i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists(benchBucketName)
b.FillPercent = options.FillPercent
for j := 0; j < options.BatchSize; j++ {
key := make([]byte, options.KeySize)
value := make([]byte, options.ValueSize)
// Write key as uint32.
binary.BigEndian.PutUint32(key, keySource())
// Insert key/value.
if err := b.Put(key, value); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
}
return nil
}
func GetSchemaFromDb(b *bolt.DB) Schema {
// first version
v := First
err := b.View(func(t *bolt.Tx) error {
b := t.Bucket(AboutBucket)
if b == nil {
return nil
}
v = VersionFromString(string(b.Get([]byte("about"))))
return nil
})
if err != nil {
return Schemas[0]
}
// find the oldest valid schema
for _, s := range Schemas {
// best case, exact match
if s.Version == v {
return s
}
// as soon as we find an older version, this is the one
if !s.Version.Greater(v) {
return s
}
}
return Schemas[0]
}
func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
for {
var count int
var top = tx.Bucket(benchBucketName)
if err := top.ForEach(func(name, _ []byte) error {
c := top.Bucket(name).Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return ErrInvalidValue
}
count++
}
return nil
}); err != nil {
return err
}
if options.WriteMode == "seq-nest" && count != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
func PrintDb(db *bolt.DB, buckets []string) error {
err := db.View(func(tx *bolt.Tx) error {
// err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
// fmt.Println(string(name))
// return nil
// })
for _, bucket := range buckets {
b := tx.Bucket([]byte(bucket))
if b == nil {
log.Printf("Bucket %v not found.", bucket)
continue
}
fmt.Println(bucket, b.Stats().KeyN)
err := b.ForEach(func(k, v []byte) error {
fmt.Printf("\t%v -> %v\n", string(k), string(v))
err := goInside(tx, v, 2)
if err != nil {
return e.Forward(err)
}
return nil
})
if err != nil {
return e.Forward(err)
}
}
return nil
})
if err != nil {
return e.Forward(err)
}
return nil
}
// walk walks recursively the bolt database db, calling walkFn for each key it finds.
func (cmd *CompactCommand) walk(db *bolt.DB, walkFn walkFunc) error {
return db.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, b *bolt.Bucket) error {
return cmd.walkBucket(b, nil, name, nil, b.Sequence(), walkFn)
})
})
}
func readMeasurePoint(db *bolt.DB, bucketName string, name string) (MeasurePoint, error) {
outMeasurePoint := MeasurePoint{}
err := db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
// if no bucket - return nil (outMeasurePoint IS empty)
if bucket == nil {
return nil
}
valBuf := bucket.Get([]byte(name))
// if no key - return nil (outMeasurePoint IS empty)
if len(valBuf) == 0 {
return nil
}
err := json.Unmarshal(valBuf, &outMeasurePoint)
if err != nil {
log.WithFields(log.Fields{
"in": "readMeasurePoint",
"after": "Unmarshal",
"ctx": "Read from Bolt",
}).Errorf("%s", err)
Stat.ErrorIncr()
return err
}
return nil
})
return outMeasurePoint, err
}
func getTotalItemFromDB(db *bolt.DB) (int64, error) {
total_item := int64(0)
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(BUCKET_STATS))
if b == nil {
return nil
}
v := b.Get([]byte(KEY_TOTALITEM))
if v == nil {
return nil
}
tmp, err := strconv.ParseInt(string(v), 10, 64)
if err != nil {
// log.Printf("ERROR: parse count %v\n", err)
return err
}
total_item = tmp
return nil
})
if err != nil {
return 0, err
}
return total_item, nil
}