func findList(db *bolt.DB, params martini.Params, req *http.Request, r render.Render) {
id := params["id"]
var dst []byte
err := db.View(
func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucketDistributionLists))
dst = b.Get([]byte(id))
return nil
})
if err != nil {
r.Error(http.StatusInternalServerError)
return
}
if dst == nil {
r.Error(http.StatusNotFound)
return
}
var l DistributionList
err = json.Unmarshal(dst, &l)
if err != nil {
r.Error(http.StatusInternalServerError)
return
}
r.JSON(http.StatusOK, l)
}
func ShowStatus(db *bolt.DB, u string) error {
ur, err := url.Parse(u)
if err != nil {
return err
}
fmt.Printf("Host %s status:\n", ur.Host)
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(ur.Host))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var sr StatRecord
err := json.Unmarshal(v, &sr)
if err != nil {
fmt.Println(err)
c.Next()
}
fmt.Printf("%s => %s\n", k, sr)
}
fmt.Printf("\n\n")
return nil
})
return nil
}
func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
var t = time.Now()
for {
c := tx.Bucket(benchBucketName).Cursor()
var count int
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return errors.New("invalid value")
}
count++
}
if options.WriteMode == "seq" && count != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
func NewMockAllocator(db *bolt.DB) *MockAllocator {
d := &MockAllocator{}
d.clustermap = make(map[string]sort.StringSlice)
var clusters []string
err := db.View(func(tx *bolt.Tx) error {
var err error
clusters, err = ClusterList(tx)
if err != nil {
return err
}
for _, cluster := range clusters {
err := d.addDevicesFromDb(tx, cluster)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return nil
}
return d
}
func DoRead(db *bolt.DB) error {
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
log.Println("b is %s", b)
//if err != nil {
// return err
//}
v := b.Get([]byte("answer"))
//log.Printf("v=%s", v)
//if err != nil {
// return err
//}
//ctrl, err := JobControlFromBytes(v)
ctrl := &JobControl{}
err := ctrl.FromBytes(v)
if err != nil {
log.Println("error", err)
return err
}
log.Printf("the value is %+v", ctrl)
return nil
})
return err
}
// QueryNodeByTaxID querys Node by taxid
func QueryNodeByTaxID(db *bolt.DB, bucket string, taxids []string) ([]nodes.Node, error) {
for _, taxid := range taxids {
if !reDigitals.MatchString(taxid) {
return []nodes.Node{}, fmt.Errorf("non-digital taxid given: %s", taxid)
}
}
nods := make([]nodes.Node, len(taxids))
if len(taxids) == 0 {
return nods, nil
}
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("database not exists: %s", bucket)
}
for i, taxid := range taxids {
s := string(b.Get([]byte(taxid)))
if s == "" {
nods[i] = nodes.Node{}
continue
}
node, err := nodes.NodeFromJSON(s)
if err != nil {
return errors.New("failed to parse node record from database")
}
nods[i] = node
}
return nil
})
return nods, err
}
func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
for {
var count int
var top = tx.Bucket(benchBucketName)
if err := top.ForEach(func(name, _ []byte) error {
c := top.Bucket(name).Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return ErrInvalidValue
}
count++
}
return nil
}); err != nil {
return err
}
if options.WriteMode == "seq-nest" && count != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
func (b *BrickEntry) DestroyCheck(db *bolt.DB, executor executors.Executor) error {
godbc.Require(db != nil)
godbc.Require(b.TpSize > 0)
godbc.Require(b.Info.Size > 0)
// Get node hostname
var host string
err := db.View(func(tx *bolt.Tx) error {
node, err := NewNodeEntryFromId(tx, b.Info.NodeId)
if err != nil {
return err
}
host = node.ManageHostName()
godbc.Check(host != "")
return nil
})
if err != nil {
return err
}
// Create request
req := &executors.BrickRequest{}
req.Name = b.Info.Id
req.Size = b.Info.Size
req.TpSize = b.TpSize
req.VgId = b.Info.DeviceId
// Check brick on node
return executor.BrickDestroyCheck(host, req)
}
func findReadAndWritePointers(db *bolt.DB) (int64, int64, error) {
writePointer := int64(0)
readPointer := int64(math.MaxInt64)
err := db.View(func(tx *bolt.Tx) error {
events := tx.Bucket(EVENTS_BUCKET)
err := events.ForEach(func(k, v []byte) error {
seq, _ := readSequence(k)
if seq > writePointer {
writePointer = seq
}
if seq < readPointer {
readPointer = seq
}
return nil
})
return err
})
if err != nil {
return -1, -1, err
}
if readPointer == int64(math.MaxInt64) {
readPointer = int64(1)
}
return writePointer, readPointer, nil
}
func ValidateLogin(user, password string, db *bolt.DB) bool {
err := db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("users"))
if bucket == nil {
return fmt.Errorf("Bucket pastes not found!")
}
var uByte []byte
if uByte = bucket.Get([]byte(user)); uByte == nil {
return fmt.Errorf("User not found!")
}
var u User
err := json.Unmarshal(uByte, &u)
if err != nil {
return err
}
err = bcrypt.CompareHashAndPassword(u.Password, []byte(password))
return err
})
if err != nil {
return false
}
return true
}
func getGroup(db *bolt.DB, id string) (g *Group, err error) {
err = db.View(func(tx *bolt.Tx) error {
g, err = getGroupWithTx(tx, id)
return err
})
return
}
func searchCityNames(
db *bolt.DB, locales []string, query string, limit int,
) (*CityNames, error) {
var cityNames CityNames
err := db.View(func(tx *bolt.Tx) error {
var err error
c := tx.Bucket(CityNamesBucketName).Cursor()
prefix := []byte(PrepareCityNameKey(query))
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
var cityName *CityName
cityName, err = CityNameFromString(string(k), string(v))
cityNames = append(cityNames, cityName)
}
return err
})
sort.Sort(CityNamesComparator{cityNames, locales})
cityNames.Uniq()
cityNames.Limit(limit)
return &cityNames, err
}
// iterate continually loops over a subsection of the database and reads key/values.
func iterate(db *bolt.DB, c chan bool) {
max := make([]byte, keySize)
binary.BigEndian.PutUint64(max, uint64(itemCount*iteratePct))
var d time.Duration
var n int
loop:
for {
t := time.Now()
// Loop over a subset of the data.
var count int
db.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte("root")).Cursor()
for k, _ := c.First(); k != nil && bytes.Compare(k, max) == -1; k, _ = c.Next() {
count++
}
return nil
})
log.Printf(" iterate: %v (n=%d)", time.Since(t), count)
d += time.Since(t)
n++
// Check for completion.
select {
case <-c:
break loop
default:
}
}
fmt.Printf("iterate: avg: %v (n=%d)\n", (d / time.Duration(n)), n)
}
// stat prints out stats about the db.
func stat(db *bolt.DB) error {
return db.View(func(tx *bolt.Tx) error {
fmt.Printf("size: %d bytes\n", tx.Size())
fmt.Println("")
return nil
})
}
func printDBDebug(db *bolt.DB) {
dbBucketName := "alphazero"
dbBucketLogs := fmt.Sprintf("%s.logs", dbBucketName)
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(dbBucketLogs))
c := b.Cursor()
i := 10
for k, v := c.Last(); k != nil; k, v = c.Prev() {
log := LogEntry{}
json.Unmarshal(v, &log)
fmt.Printf("- %s\n", log.String())
i--
if i == 0 {
break
}
}
return nil
})
dbBucketStats := fmt.Sprintf("%s.stats", dbBucketName)
db.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(dbBucketStats)).Cursor()
prefix := []byte("stats.")
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
fmt.Printf("key=%s, value=%s\n", k, v)
}
return nil
})
}
// QueryDivisionByDivisionID querys Division by taxid
func QueryDivisionByDivisionID(db *bolt.DB, bucket string, ids []string) ([]nodes.Division, error) {
for _, id := range ids {
if !reDigitals.MatchString(id) {
return []nodes.Division{}, fmt.Errorf("non-digital division given: %s", id)
}
}
divisions := make([]nodes.Division, len(ids))
if len(ids) == 0 {
return divisions, nil
}
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("database not exists: %s", bucket)
}
for i, id := range ids {
s := string(b.Get([]byte(id)))
if s == "" {
divisions[i] = nodes.Division{}
continue
}
division, err := nodes.DivisionFromJSON(s)
if err != nil {
return errors.New("failed to parse division record from database")
}
divisions[i] = division
}
return nil
})
return divisions, err
}
// LoadAllNodes loads all nodes into memory
func LoadAllNodes(db *bolt.DB, bucket string) (map[string]nodes.Node, error) {
nods := make(map[string]nodes.Node)
ch := make(chan string, runtime.NumCPU())
chDone := make(chan int)
go func() {
for s := range ch {
node, err := nodes.NodeFromJSON(s)
checkError(err)
nods[node.TaxID] = node
}
chDone <- 1
}()
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucket))
if b == nil {
return fmt.Errorf("database not exists: %s", bucket)
}
b.ForEach(func(k, v []byte) error {
ch <- string(v)
return nil
})
return nil
})
close(ch)
<-chDone
return nods, err
}
func readItems(db *bolt.DB) map[string]godragon.Item {
itemm := make(map[string]godragon.Item)
db.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte("items")).Cursor()
var buff bytes.Buffer
for k, v := c.First(); k != nil; k, v = c.Next() {
var item godragon.Item
dec := gob.NewDecoder(&buff)
buff.Write(v)
err := dec.Decode(&item)
if err != nil {
log.Println(err)
return err
}
itemm[string(k)] = item
buff.Reset()
}
return nil
})
return itemm
}
// Exists checks if the key exists inside the store
func (b *BoltDB) Exists(key string) (bool, error) {
var (
val []byte
db *bolt.DB
err error
)
b.Lock()
defer b.Unlock()
if db, err = b.getDBhandle(); err != nil {
return false, err
}
defer b.releaseDBhandle()
err = db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(b.boltBucket)
if bucket == nil {
return store.ErrKeyNotFound
}
val = bucket.Get([]byte(key))
return nil
})
if len(val) == 0 {
return false, err
}
return true, err
}
func viewAll(db *bolt.DB) {
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("Shows"))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
show, err := ShowFromBytes(v)
if err != nil {
fmt.Println("Error loading show '%s'", k)
continue
}
name := show.Name
hasNext := ""
isPaused := ""
if len(name) > 15 {
name = name[:15] + "..."
}
if _, ok := show.NextPointer(); ok {
hasNext = "+"
}
if show.Paused {
isPaused = "paused"
}
fmt.Printf("%-20s %s %s %s\n", name, show.Pointer, hasNext, isPaused)
}
return nil
})
}
// walk walks recursively the bolt database db, calling walkFn for each key it finds.
func (cmd *CompactCommand) walk(db *bolt.DB, walkFn walkFunc) error {
return db.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, b *bolt.Bucket) error {
return cmd.walkBucket(b, nil, name, nil, b.Sequence(), walkFn)
})
})
}
func view(db *bolt.DB, name string) {
var (
err error
show *Show
)
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("Shows"))
show, err = ShowFromBytes(b.Get([]byte(name)))
return err
})
if show == nil {
fmt.Printf("The specified show '%s' not found.\n", name)
return
}
fmt.Printf("Name : %s\n", show.Name)
fmt.Printf("Query : %s\n", show.Query)
fmt.Printf("Min seeders : %d\n", show.SeedersMin)
fmt.Printf("Prefer HQ : %t\n", show.PreferHQ)
fmt.Printf("Pointer : %s\n", show.Pointer)
fmt.Printf("Paused : %t\n", show.Paused)
seasons := show.Seasons.Slice()
if len(seasons) > 0 {
fmt.Println("")
fmt.Println("Season Episodes Begin")
for _, season := range seasons {
fmt.Printf("%-7d %-9d %s\n", season.Number, season.EpisodeCount, season.Begin)
}
}
}
func addUser(db *bolt.DB, email string, password string) bool {
check := []byte("")
hashedPass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
fmt.Println(err)
}
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("UsersBucket"))
check = b.Get([]byte(email))
return nil
})
if check == nil {
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("UsersBucket"))
err := b.Put([]byte(email), []byte(hashedPass))
return err
})
if err != nil {
fmt.Println(err)
}
return true
} else {
return false
}
}
func GetStatistics(db *bolt.DB) (*Statistics, error) {
var stat Statistics
err := db.View(func(tx *bolt.Tx) error {
var err error
if bucket := tx.Bucket(StatisticsBucketName); bucket != nil {
stat.CountriesCount, err = strconv.Atoi(
string(bucket.Get([]byte("countries_count"))),
)
stat.CitiesCount, err = strconv.Atoi(
string(bucket.Get([]byte("cities_count"))),
)
stat.CityNamesCount, err = strconv.Atoi(
string(bucket.Get([]byte("city_names_count"))),
)
}
if err == nil && stat.CityNamesCount > 0 {
stat.Status = "ok"
} else {
stat.Status = "indexing"
}
return err
})
return &stat, err
}
func GetSchemaFromDb(b *bolt.DB) Schema {
// first version
v := First
err := b.View(func(t *bolt.Tx) error {
b := t.Bucket(AboutBucket)
if b == nil {
return nil
}
v = VersionFromString(string(b.Get([]byte("about"))))
return nil
})
if err != nil {
return Schemas[0]
}
// find the oldest valid schema
for _, s := range Schemas {
// best case, exact match
if s.Version == v {
return s
}
// as soon as we find an older version, this is the one
if !s.Version.Greater(v) {
return s
}
}
return Schemas[0]
}
func executeSelect(stmt *boltq.SelectStatement, db *bolt.DB) error {
return db.View(func(tx *bolt.Tx) error {
var bucket *bolt.Bucket
for _, name := range stmt.BucketPath {
log.Debugln("navigating to bucket", name)
bucket = tx.Bucket([]byte(name))
if bucket == nil {
return fmt.Errorf("cannot find bucket %s", name)
}
}
if containsAsterisk(stmt.Fields) {
log.Debugln("interating keys")
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
emitKeypair(k, v)
}
} else {
for _, k := range stmt.Fields {
keyBytes := []byte(k)
v := bucket.Get(keyBytes)
emitKeypair(keyBytes, v)
}
}
return nil
})
}
func PrintDb(db *bolt.DB, buckets []string) error {
err := db.View(func(tx *bolt.Tx) error {
// err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
// fmt.Println(string(name))
// return nil
// })
for _, bucket := range buckets {
b := tx.Bucket([]byte(bucket))
if b == nil {
log.Printf("Bucket %v not found.", bucket)
continue
}
fmt.Println(bucket, b.Stats().KeyN)
err := b.ForEach(func(k, v []byte) error {
fmt.Printf("\t%v -> %v\n", string(k), string(v))
err := goInside(tx, v, 2)
if err != nil {
return e.Forward(err)
}
return nil
})
if err != nil {
return e.Forward(err)
}
}
return nil
})
if err != nil {
return e.Forward(err)
}
return nil
}
func getTotalItemFromDB(db *bolt.DB) (int64, error) {
total_item := int64(0)
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(BUCKET_STATS))
if b == nil {
return nil
}
v := b.Get([]byte(KEY_TOTALITEM))
if v == nil {
return nil
}
tmp, err := strconv.ParseInt(string(v), 10, 64)
if err != nil {
// log.Printf("ERROR: parse count %v\n", err)
return err
}
total_item = tmp
return nil
})
if err != nil {
return 0, err
}
return total_item, nil
}
func readMeasurePoint(db *bolt.DB, bucketName string, name string) (MeasurePoint, error) {
outMeasurePoint := MeasurePoint{}
err := db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
// if no bucket - return nil (outMeasurePoint IS empty)
if bucket == nil {
return nil
}
valBuf := bucket.Get([]byte(name))
// if no key - return nil (outMeasurePoint IS empty)
if len(valBuf) == 0 {
return nil
}
err := json.Unmarshal(valBuf, &outMeasurePoint)
if err != nil {
log.WithFields(log.Fields{
"in": "readMeasurePoint",
"after": "Unmarshal",
"ctx": "Read from Bolt",
}).Errorf("%s", err)
Stat.ErrorIncr()
return err
}
return nil
})
return outMeasurePoint, err
}
func allLists(db *bolt.DB, params martini.Params, req *http.Request, r render.Render) {
var lists []DistributionList
err := db.View(
func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucketDistributionLists))
return b.ForEach(func(k, v []byte) error {
var d DistributionList
err := json.Unmarshal(v, &d)
// TODO(dgryski): skip invalid keys instead of aborting on corrupt db?[
if err != nil {
return err
}
lists = append(lists, d)
return nil
})
})
if err != nil {
r.Error(http.StatusInternalServerError)
return
}
r.JSON(http.StatusOK, lists)
}
