Esempio n. 1
0
// AtomicDelete deletes a value at "key" if the key
// has not been modified in the meantime, throws an
// error if this is the case
func (b *BoltDB) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
	var (
		val []byte
		db  *bolt.DB
		err error
	)

	if previous == nil {
		return false, store.ErrPreviousNotSpecified
	}
	if db, err = b.getDBhandle(); err != nil {
		return false, err
	}
	defer b.releaseDBhandle()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(b.boltBucket)
		if bucket == nil {
			return ErrBoltBucketNotFound
		}

		val = bucket.Get([]byte(key))
		dbIndex := binary.LittleEndian.Uint64(val[:libkvmetadatalen])
		if dbIndex != previous.LastIndex {
			return store.ErrKeyModified
		}
		err := bucket.Delete([]byte(key))
		return err
	})
	if err != nil {
		return false, err
	}
	return true, err
}
Esempio n. 2
0
func ShowStatus(db *bolt.DB, u string) error {
	ur, err := url.Parse(u)
	if err != nil {
		return err
	}

	fmt.Printf("Host %s status:\n", ur.Host)
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(ur.Host))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var sr StatRecord
			err := json.Unmarshal(v, &sr)
			if err != nil {
				fmt.Println(err)
				c.Next()
			}

			fmt.Printf("%s => %s\n", k, sr)
		}
		fmt.Printf("\n\n")
		return nil
	})
	return nil
}
Esempio n. 3
0
func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
	return db.View(func(tx *bolt.Tx) error {
		var t = time.Now()

		for {
			c := tx.Bucket(benchBucketName).Cursor()
			var count int
			for k, v := c.First(); k != nil; k, v = c.Next() {
				if v == nil {
					return errors.New("invalid value")
				}
				count++
			}

			if options.WriteMode == "seq" && count != options.Iterations {
				return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
			}

			results.ReadOps += count

			// Make sure we do this for at least a second.
			if time.Since(t) >= time.Second {
				break
			}
		}

		return nil
	})
}
Esempio n. 4
0
// init initializes the database for this layer setup.
func (l *Layered) init(db *bolt.DB) ([]*layerVertex, error) {
	layerVertices := make([]*layerVertex, len(l.Layers))
	for i, layer := range l.Layers {
		var parent *Layer
		if i > 0 {
			parent = l.Layers[i-1]
		}

		layerVertex, err := l.initLayer(db, layer, parent)
		if err != nil {
			return nil, err
		}

		layerVertices[i] = layerVertex
		if parent != nil {
			// We have a prior layer, so setup the edge pointer
			err = db.Update(func(tx *bolt.Tx) error {
				bucket := tx.Bucket(boltEdgesBucket)
				return bucket.Put(
					[]byte(layer.ID),
					[]byte(parent.ID))
			})
			if err != nil {
				return nil, err
			}
		}
	}

	return layerVertices, nil
}
Esempio n. 5
0
func (l *Layered) deleteLayer(db *bolt.DB, layer *Layer, path string) error {
	if err := os.RemoveAll(path); err != nil {
		return err
	}

	return db.Update(func(tx *bolt.Tx) error {
		// Delete the layer itself
		bucket := tx.Bucket(boltLayersBucket)
		key := []byte(layer.ID)
		if err := bucket.Delete(key); err != nil {
			return err
		}

		// Delete all the edges
		bucket = tx.Bucket(boltEdgesBucket)
		if err := bucket.Delete(key); err != nil {
			return err
		}

		// Find any values
		return bucket.ForEach(func(k, data []byte) error {
			if string(data) == layer.ID {
				return bucket.Delete(k)
			}

			return nil
		})
	})
}
Esempio n. 6
0
func benchWriteNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
	results.WriteOps = options.Iterations

	for i := 0; i < options.Iterations; i += options.BatchSize {
		err := db.Update(func(tx *bolt.Tx) error {
			top, _ := tx.CreateBucketIfNotExists(benchBucketName)

			var name = make([]byte, options.KeySize)
			binary.BigEndian.PutUint32(name, keySource())
			b, _ := top.CreateBucketIfNotExists(name)

			for j := 0; j < options.BatchSize; j++ {
				var key = make([]byte, options.KeySize)
				var value = make([]byte, options.ValueSize)
				binary.BigEndian.PutUint32(key, keySource())
				if err := b.Put(key, value); err != nil {
					return err
				}
			}

			return nil
		})
		if err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 7
0
func addUser(db *bolt.DB, email string, password string) bool {
	check := []byte("")
	hashedPass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		fmt.Println(err)
	}
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("UsersBucket"))
		check = b.Get([]byte(email))
		return nil
	})
	if check == nil {
		db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte("UsersBucket"))
			err := b.Put([]byte(email), []byte(hashedPass))
			return err
		})
		if err != nil {
			fmt.Println(err)
		}
		return true
	} else {
		return false
	}
}
Esempio n. 8
0
// QueryNodeByTaxID querys Node by taxid
func QueryNodeByTaxID(db *bolt.DB, bucket string, taxids []string) ([]nodes.Node, error) {
	for _, taxid := range taxids {
		if !reDigitals.MatchString(taxid) {
			return []nodes.Node{}, fmt.Errorf("non-digital taxid given: %s", taxid)
		}
	}
	nods := make([]nodes.Node, len(taxids))
	if len(taxids) == 0 {
		return nods, nil
	}
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if b == nil {
			return fmt.Errorf("database not exists: %s", bucket)
		}
		for i, taxid := range taxids {
			s := string(b.Get([]byte(taxid)))
			if s == "" {
				nods[i] = nodes.Node{}
				continue
			}
			node, err := nodes.NodeFromJSON(s)
			if err != nil {
				return errors.New("failed to parse node record from database")
			}
			nods[i] = node
		}
		return nil
	})
	return nods, err
}
Esempio n. 9
0
func (s *server) dump(db *bolt.DB, changes map[uint64]bool) {
	for k := range changes {
		// marshal
		var rs *RankSet
		s.lock_read(func() {
			rs = s.ranks[k]
		})
		if rs == nil {
			log.Warning("empty rankset:", k)
			continue
		}

		// serialization and save
		bin, err := rs.Marshal()
		if err != nil {
			log.Critical("cannot marshal:", err)
			os.Exit(-1)
		}

		db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(BOLTDB_BUCKET))
			err := b.Put([]byte(fmt.Sprint(k)), bin)
			return err
		})
	}
}
Esempio n. 10
0
// *whirrrrrrrrrrr*
func probe(db *bolt.DB, api *tbotapi.TelegramBotAPI, closed chan struct{}, wg *sync.WaitGroup) {
	dbBucketName := "alphazero"
	dbBucketStats := fmt.Sprintf("%s.stats", dbBucketName)
	for {
		select {
		case <-closed:
			wg.Done()
			return
		case val := <-api.Updates:
			typ := val.Message.Type()
			if typ != model.TextType {
				fmt.Fprintf(os.Stderr, "Not handling type: %s\n", typ)
				continue
			}

			response, md := process(db, val)
			db.Update(func(tx *bolt.Tx) error {
				b := tx.Bucket([]byte(dbBucketStats))
				id := []byte("messages.count.received")
				var val []byte
				if val = b.Get(id); val == nil {
					val = []byte("0")
				}
				c, _ := strconv.Atoi(string(val))
				c++
				return b.Put(id, []byte(strconv.Itoa(c)))
			})

			// unsafeval := blackfriday.MarkdownCommon([]byte(*val.Message.Text))
			// unsafe := blackfriday.MarkdownCommon([]byte(response))
			// html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)

			sender := val.Message.Chat
			recipient := model.NewRecipientFromChat(val.Message.Chat)

			om := model.NewOutgoingMessage(
				model.NewChatRecipient(*recipient.ChatID),
				response).SetMarkdown(md)

			var msg *model.MessageResponse
			var err error

			msg, err = api.SendMessageExtended(om)

			if err != nil {
				msg, err = api.SendMessageExtended(om.SetMarkdown(false))
				if err != nil {
					fmt.Fprintf(os.Stderr, "Err: %s\n", err)
					continue
				}
			}

			fmt.Printf(
				"MessageID: %04d, ChatID: %s, Text: %s, IsGroupChat:%t\n",
				msg.Message.ID, sender.String(), response, msg.Message.Chat.IsGroupChat())
		case val := <-api.Errors:
			fmt.Printf("Err: %s\n", val)
		}
	}
}
Esempio n. 11
0
// LoadAllNodes loads all nodes into memory
func LoadAllNodes(db *bolt.DB, bucket string) (map[string]nodes.Node, error) {
	nods := make(map[string]nodes.Node)

	ch := make(chan string, runtime.NumCPU())
	chDone := make(chan int)
	go func() {
		for s := range ch {
			node, err := nodes.NodeFromJSON(s)
			checkError(err)
			nods[node.TaxID] = node
		}
		chDone <- 1
	}()

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if b == nil {
			return fmt.Errorf("database not exists: %s", bucket)
		}

		b.ForEach(func(k, v []byte) error {
			ch <- string(v)
			return nil
		})

		return nil
	})
	close(ch)
	<-chDone
	return nods, err
}
Esempio n. 12
0
func printDBDebug(db *bolt.DB) {
	dbBucketName := "alphazero"
	dbBucketLogs := fmt.Sprintf("%s.logs", dbBucketName)
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(dbBucketLogs))
		c := b.Cursor()
		i := 10
		for k, v := c.Last(); k != nil; k, v = c.Prev() {
			log := LogEntry{}
			json.Unmarshal(v, &log)
			fmt.Printf("- %s\n", log.String())
			i--
			if i == 0 {
				break
			}
		}
		return nil
	})

	dbBucketStats := fmt.Sprintf("%s.stats", dbBucketName)
	db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte(dbBucketStats)).Cursor()
		prefix := []byte("stats.")
		for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
			fmt.Printf("key=%s, value=%s\n", k, v)
		}
		return nil
	})
}
Esempio n. 13
0
func logAction(db *bolt.DB, action string) {
	dbBucketName := "alphazero"
	dbBucketLogs := fmt.Sprintf("%s.logs", dbBucketName)
	db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(dbBucketLogs))

		id, _ := b.NextSequence()
		l := LogEntry{
			ID:     id,
			When:   strconv.FormatInt(time.Now().Unix(), 10),
			Action: action,
			UUID:   uuid.NewV4().String(),
		}
		buf, err := json.Marshal(l)
		if err != nil {
			return err
		}

		err = b.Put(itob(l.ID), buf)
		if err != nil {
			return err
		}
		return nil
	})
}
Esempio n. 14
0
func (c *Context) SerializeIndex(index *Index) (err error) {
	var data []byte
	var db *bolt.DB

	if data, err = json.Marshal(index); err != nil {
		return
	}

	db, err = c.OpenDB()
	if err != nil {
		return err
	}
	defer db.Close()

	return db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(byteify(IndicesKey))
		if err != nil {
			return err
		}
		if bucket == nil {
			return ErrNoSuchDbBucket
		}
		return bucket.Put(byteify(index.FileId), data)
	})
}
Esempio n. 15
0
// newBroadcastsStore creates a new BroadcastAddresses object after doing the
// necessary initialization.
func newBroadcastsStore(db *bolt.DB, username string) (*BroadcastAddresses, error) {

	b := &BroadcastAddresses{
		db:       db,
		username: []byte(username),
		addrs:    make([]bmutil.Address, 0),
	}

	err := db.Update(func(tx *bolt.Tx) error {
		userbucket, err := tx.CreateBucketIfNotExists([]byte(username))
		if err != nil {
			return err
		}

		bucket, err := userbucket.CreateBucketIfNotExists(broadcastAddressesBucket)
		if err != nil {
			return err
		}

		return bucket.ForEach(func(k, _ []byte) error {
			addr, err := bmutil.DecodeAddress(string(k))
			if err != nil {
				return err
			}

			b.addrs = append(b.addrs, *addr)
			return nil
		})
	})
	if err != nil {
		return nil, err
	}

	return b, nil
}
Esempio n. 16
0
func InsertData(db *bolt.DB, artist string, track string, cover string) {
	db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(DATABASE_BUCKET)
		err := b.Put([]byte(artist+" - "+track), []byte(cover))
		return err
	})
}
Esempio n. 17
0
func NewMockAllocator(db *bolt.DB) *MockAllocator {
	d := &MockAllocator{}
	d.clustermap = make(map[string]sort.StringSlice)

	var clusters []string
	err := db.View(func(tx *bolt.Tx) error {
		var err error
		clusters, err = ClusterList(tx)
		if err != nil {
			return err
		}

		for _, cluster := range clusters {
			err := d.addDevicesFromDb(tx, cluster)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		return nil
	}

	return d
}
Esempio n. 18
0
//Put the key, value pair. index number metadata is prepended to the value
func (b *BoltDB) Put(key string, value []byte, opts *store.WriteOptions) error {
	var (
		dbIndex uint64
		db      *bolt.DB
		err     error
	)
	b.Lock()
	defer b.Unlock()

	dbval := make([]byte, libkvmetadatalen)

	if db, err = b.getDBhandle(); err != nil {
		return err
	}
	defer b.releaseDBhandle()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(b.boltBucket)
		if err != nil {
			return err
		}

		dbIndex = atomic.AddUint64(&b.dbIndex, 1)
		binary.LittleEndian.PutUint64(dbval, dbIndex)
		dbval = append(dbval, value...)

		err = bucket.Put([]byte(key), dbval)
		if err != nil {
			return err
		}
		return nil
	})
	return err
}
Esempio n. 19
0
func writePageAttribute(PageDB *bolt.DB, AttrName, AttrValue string) error {
	UpdateFuncErr := PageDB.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("PageAttributes"))
		return bucket.Put([]byte(AttrName), []byte(AttrValue))
	})
	return UpdateFuncErr
}
Esempio n. 20
0
// Exists checks if the key exists inside the store
func (b *BoltDB) Exists(key string) (bool, error) {
	var (
		val []byte
		db  *bolt.DB
		err error
	)
	b.Lock()
	defer b.Unlock()

	if db, err = b.getDBhandle(); err != nil {
		return false, err
	}
	defer b.releaseDBhandle()

	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(b.boltBucket)
		if bucket == nil {
			return store.ErrKeyNotFound
		}

		val = bucket.Get([]byte(key))

		return nil
	})

	if len(val) == 0 {
		return false, err
	}
	return true, err
}
Esempio n. 21
0
func DoRead(db *bolt.DB) error {
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("MyBucket"))
		log.Println("b is %s", b)
		//if err != nil {
		//	return err
		//}

		v := b.Get([]byte("answer"))
		//log.Printf("v=%s", v)
		//if err != nil {
		//	return err
		//}
		//ctrl, err := JobControlFromBytes(v)
		ctrl := &JobControl{}
		err := ctrl.FromBytes(v)
		if err != nil {
			log.Println("error", err)
			return err
		}
		log.Printf("the value is %+v", ctrl)
		return nil
	})
	return err
}
Esempio n. 22
0
// DeleteTree deletes a range of keys with a given prefix
func (b *BoltDB) DeleteTree(keyPrefix string) error {
	var (
		db  *bolt.DB
		err error
	)
	b.Lock()
	defer b.Unlock()

	if db, err = b.getDBhandle(); err != nil {
		return err
	}
	defer b.releaseDBhandle()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(b.boltBucket)
		if bucket == nil {
			return store.ErrKeyNotFound
		}

		cursor := bucket.Cursor()
		prefix := []byte(keyPrefix)

		for key, _ := cursor.Seek(prefix); bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
			_ = bucket.Delete([]byte(key))
		}
		return nil
	})

	return err
}
Esempio n. 23
0
func (l *Layered) updateLayer(db *bolt.DB, layer *Layer, f func(*layerVertex)) error {
	return db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(boltLayersBucket)
		key := []byte(layer.ID)
		data := bucket.Get(key)
		if len(data) == 0 {
			// This should never happen through this struct
			panic(fmt.Errorf("layer %s not found", layer.ID))
		}

		// Read the vertex, call the function to modify it
		var v layerVertex
		if err := l.structRead(&v, data); err != nil {
			return err
		}
		f(&v)

		// Save the resulting layer data
		data, err := l.structData(&v)
		if err != nil {
			return err
		}
		return bucket.Put(key, data)
	})
}
Esempio n. 24
0
func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
	results.WriteOps = options.Iterations

	for i := 0; i < options.Iterations; i += options.BatchSize {
		if err := db.Update(func(tx *bolt.Tx) error {
			b, _ := tx.CreateBucketIfNotExists(benchBucketName)
			b.FillPercent = options.FillPercent

			for j := 0; j < options.BatchSize; j++ {
				key := make([]byte, options.KeySize)
				value := make([]byte, options.ValueSize)

				// Write key as uint32.
				binary.BigEndian.PutUint32(key, keySource())

				// Insert key/value.
				if err := b.Put(key, value); err != nil {
					return err
				}
			}

			return nil
		}); err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 25
0
func GetSchemaFromDb(b *bolt.DB) Schema {

	// first version
	v := First
	err := b.View(func(t *bolt.Tx) error {
		b := t.Bucket(AboutBucket)
		if b == nil {
			return nil
		}

		v = VersionFromString(string(b.Get([]byte("about"))))
		return nil
	})
	if err != nil {
		return Schemas[0]
	}

	// find the oldest valid schema
	for _, s := range Schemas {

		// best case, exact match
		if s.Version == v {
			return s
		}

		// as soon as we find an older version, this is the one
		if !s.Version.Greater(v) {
			return s
		}
	}

	return Schemas[0]
}
Esempio n. 26
0
func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
	return db.View(func(tx *bolt.Tx) error {
		t := time.Now()

		for {
			var count int
			var top = tx.Bucket(benchBucketName)
			if err := top.ForEach(func(name, _ []byte) error {
				c := top.Bucket(name).Cursor()
				for k, v := c.First(); k != nil; k, v = c.Next() {
					if v == nil {
						return ErrInvalidValue
					}
					count++
				}
				return nil
			}); err != nil {
				return err
			}

			if options.WriteMode == "seq-nest" && count != options.Iterations {
				return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
			}

			results.ReadOps += count

			// Make sure we do this for at least a second.
			if time.Since(t) >= time.Second {
				break
			}
		}

		return nil
	})
}
Esempio n. 27
0
func PrintDb(db *bolt.DB, buckets []string) error {
	err := db.View(func(tx *bolt.Tx) error {
		// err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// 	fmt.Println(string(name))
		// 	return nil
		// })
		for _, bucket := range buckets {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				log.Printf("Bucket %v not found.", bucket)
				continue
			}
			fmt.Println(bucket, b.Stats().KeyN)
			err := b.ForEach(func(k, v []byte) error {
				fmt.Printf("\t%v -> %v\n", string(k), string(v))
				err := goInside(tx, v, 2)
				if err != nil {
					return e.Forward(err)
				}
				return nil
			})
			if err != nil {
				return e.Forward(err)
			}
		}
		return nil
	})
	if err != nil {
		return e.Forward(err)
	}
	return nil
}
Esempio n. 28
0
// walk walks recursively the bolt database db, calling walkFn for each key it finds.
func (cmd *CompactCommand) walk(db *bolt.DB, walkFn walkFunc) error {
	return db.View(func(tx *bolt.Tx) error {
		return tx.ForEach(func(name []byte, b *bolt.Bucket) error {
			return cmd.walkBucket(b, nil, name, nil, b.Sequence(), walkFn)
		})
	})
}
Esempio n. 29
0
func readMeasurePoint(db *bolt.DB, bucketName string, name string) (MeasurePoint, error) {

	outMeasurePoint := MeasurePoint{}
	err := db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		// if no bucket - return nil (outMeasurePoint IS empty)
		if bucket == nil {
			return nil
		}

		valBuf := bucket.Get([]byte(name))
		// if no key - return nil (outMeasurePoint IS empty)
		if len(valBuf) == 0 {
			return nil
		}

		err := json.Unmarshal(valBuf, &outMeasurePoint)
		if err != nil {
			log.WithFields(log.Fields{
				"in":    "readMeasurePoint",
				"after": "Unmarshal",
				"ctx":   "Read from Bolt",
			}).Errorf("%s", err)
			Stat.ErrorIncr()
			return err
		}
		return nil
	})

	return outMeasurePoint, err
}
Esempio n. 30
0
func getTotalItemFromDB(db *bolt.DB) (int64, error) {
	total_item := int64(0)
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(BUCKET_STATS))
		if b == nil {
			return nil
		}
		v := b.Get([]byte(KEY_TOTALITEM))
		if v == nil {
			return nil
		}
		tmp, err := strconv.ParseInt(string(v), 10, 64)
		if err != nil {
			// log.Printf("ERROR: parse count %v\n", err)
			return err
		}

		total_item = tmp
		return nil
	})
	if err != nil {
		return 0, err
	}
	return total_item, nil
}