Example #1
0
func findList(db *bolt.DB, params martini.Params, req *http.Request, r render.Render) {
	id := params["id"]

	var dst []byte
	err := db.View(
		func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucketDistributionLists))
			dst = b.Get([]byte(id))
			return nil
		})

	if err != nil {
		r.Error(http.StatusInternalServerError)
		return
	}

	if dst == nil {
		r.Error(http.StatusNotFound)
		return
	}

	var l DistributionList
	err = json.Unmarshal(dst, &l)

	if err != nil {
		r.Error(http.StatusInternalServerError)
		return
	}

	r.JSON(http.StatusOK, l)
}
Example #2
0
func ShowStatus(db *bolt.DB, u string) error {
	ur, err := url.Parse(u)
	if err != nil {
		return err
	}

	fmt.Printf("Host %s status:\n", ur.Host)
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(ur.Host))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var sr StatRecord
			err := json.Unmarshal(v, &sr)
			if err != nil {
				fmt.Println(err)
				c.Next()
			}

			fmt.Printf("%s => %s\n", k, sr)
		}
		fmt.Printf("\n\n")
		return nil
	})
	return nil
}
Example #3
0
func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
	return db.View(func(tx *bolt.Tx) error {
		var t = time.Now()

		for {
			c := tx.Bucket(benchBucketName).Cursor()
			var count int
			for k, v := c.First(); k != nil; k, v = c.Next() {
				if v == nil {
					return errors.New("invalid value")
				}
				count++
			}

			if options.WriteMode == "seq" && count != options.Iterations {
				return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
			}

			results.ReadOps += count

			// Make sure we do this for at least a second.
			if time.Since(t) >= time.Second {
				break
			}
		}

		return nil
	})
}
Example #4
0
func NewMockAllocator(db *bolt.DB) *MockAllocator {
	d := &MockAllocator{}
	d.clustermap = make(map[string]sort.StringSlice)

	var clusters []string
	err := db.View(func(tx *bolt.Tx) error {
		var err error
		clusters, err = ClusterList(tx)
		if err != nil {
			return err
		}

		for _, cluster := range clusters {
			err := d.addDevicesFromDb(tx, cluster)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		return nil
	}

	return d
}
func DoRead(db *bolt.DB) error {
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("MyBucket"))
		log.Println("b is %s", b)
		//if err != nil {
		//	return err
		//}

		v := b.Get([]byte("answer"))
		//log.Printf("v=%s", v)
		//if err != nil {
		//	return err
		//}
		//ctrl, err := JobControlFromBytes(v)
		ctrl := &JobControl{}
		err := ctrl.FromBytes(v)
		if err != nil {
			log.Println("error", err)
			return err
		}
		log.Printf("the value is %+v", ctrl)
		return nil
	})
	return err
}
Example #6
0
// QueryNodeByTaxID querys Node by taxid
func QueryNodeByTaxID(db *bolt.DB, bucket string, taxids []string) ([]nodes.Node, error) {
	for _, taxid := range taxids {
		if !reDigitals.MatchString(taxid) {
			return []nodes.Node{}, fmt.Errorf("non-digital taxid given: %s", taxid)
		}
	}
	nods := make([]nodes.Node, len(taxids))
	if len(taxids) == 0 {
		return nods, nil
	}
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if b == nil {
			return fmt.Errorf("database not exists: %s", bucket)
		}
		for i, taxid := range taxids {
			s := string(b.Get([]byte(taxid)))
			if s == "" {
				nods[i] = nodes.Node{}
				continue
			}
			node, err := nodes.NodeFromJSON(s)
			if err != nil {
				return errors.New("failed to parse node record from database")
			}
			nods[i] = node
		}
		return nil
	})
	return nods, err
}
Example #7
0
func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
	return db.View(func(tx *bolt.Tx) error {
		t := time.Now()

		for {
			var count int
			var top = tx.Bucket(benchBucketName)
			if err := top.ForEach(func(name, _ []byte) error {
				c := top.Bucket(name).Cursor()
				for k, v := c.First(); k != nil; k, v = c.Next() {
					if v == nil {
						return ErrInvalidValue
					}
					count++
				}
				return nil
			}); err != nil {
				return err
			}

			if options.WriteMode == "seq-nest" && count != options.Iterations {
				return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
			}

			results.ReadOps += count

			// Make sure we do this for at least a second.
			if time.Since(t) >= time.Second {
				break
			}
		}

		return nil
	})
}
Example #8
0
func (b *BrickEntry) DestroyCheck(db *bolt.DB, executor executors.Executor) error {
	godbc.Require(db != nil)
	godbc.Require(b.TpSize > 0)
	godbc.Require(b.Info.Size > 0)

	// Get node hostname
	var host string
	err := db.View(func(tx *bolt.Tx) error {
		node, err := NewNodeEntryFromId(tx, b.Info.NodeId)
		if err != nil {
			return err
		}

		host = node.ManageHostName()
		godbc.Check(host != "")
		return nil
	})
	if err != nil {
		return err
	}

	// Create request
	req := &executors.BrickRequest{}
	req.Name = b.Info.Id
	req.Size = b.Info.Size
	req.TpSize = b.TpSize
	req.VgId = b.Info.DeviceId

	// Check brick on node
	return executor.BrickDestroyCheck(host, req)
}
Example #9
0
func findReadAndWritePointers(db *bolt.DB) (int64, int64, error) {
	writePointer := int64(0)
	readPointer := int64(math.MaxInt64)
	err := db.View(func(tx *bolt.Tx) error {
		events := tx.Bucket(EVENTS_BUCKET)
		err := events.ForEach(func(k, v []byte) error {
			seq, _ := readSequence(k)
			if seq > writePointer {
				writePointer = seq
			}
			if seq < readPointer {
				readPointer = seq
			}
			return nil
		})

		return err
	})

	if err != nil {
		return -1, -1, err
	}

	if readPointer == int64(math.MaxInt64) {
		readPointer = int64(1)
	}

	return writePointer, readPointer, nil
}
Example #10
0
func ValidateLogin(user, password string, db *bolt.DB) bool {
	err := db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte("users"))
		if bucket == nil {
			return fmt.Errorf("Bucket pastes not found!")
		}

		var uByte []byte
		if uByte = bucket.Get([]byte(user)); uByte == nil {
			return fmt.Errorf("User not found!")
		}

		var u User
		err := json.Unmarshal(uByte, &u)
		if err != nil {
			return err
		}

		err = bcrypt.CompareHashAndPassword(u.Password, []byte(password))
		return err
	})
	if err != nil {
		return false
	}
	return true
}
Example #11
0
func getGroup(db *bolt.DB, id string) (g *Group, err error) {
	err = db.View(func(tx *bolt.Tx) error {
		g, err = getGroupWithTx(tx, id)
		return err
	})
	return
}
Example #12
0
func searchCityNames(
	db *bolt.DB, locales []string, query string, limit int,
) (*CityNames, error) {
	var cityNames CityNames

	err := db.View(func(tx *bolt.Tx) error {
		var err error
		c := tx.Bucket(CityNamesBucketName).Cursor()

		prefix := []byte(PrepareCityNameKey(query))
		for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
			var cityName *CityName
			cityName, err = CityNameFromString(string(k), string(v))
			cityNames = append(cityNames, cityName)
		}

		return err
	})

	sort.Sort(CityNamesComparator{cityNames, locales})
	cityNames.Uniq()
	cityNames.Limit(limit)

	return &cityNames, err
}
Example #13
0
// iterate continually loops over a subsection of the database and reads key/values.
func iterate(db *bolt.DB, c chan bool) {
	max := make([]byte, keySize)
	binary.BigEndian.PutUint64(max, uint64(itemCount*iteratePct))

	var d time.Duration
	var n int
loop:
	for {
		t := time.Now()

		// Loop over a subset of the data.
		var count int
		db.View(func(tx *bolt.Tx) error {
			c := tx.Bucket([]byte("root")).Cursor()
			for k, _ := c.First(); k != nil && bytes.Compare(k, max) == -1; k, _ = c.Next() {
				count++
			}
			return nil
		})
		log.Printf("  iterate: %v (n=%d)", time.Since(t), count)
		d += time.Since(t)
		n++

		// Check for completion.
		select {
		case <-c:
			break loop
		default:
		}
	}

	fmt.Printf("iterate: avg: %v (n=%d)\n", (d / time.Duration(n)), n)
}
Example #14
0
// stat prints out stats about the db.
func stat(db *bolt.DB) error {
	return db.View(func(tx *bolt.Tx) error {
		fmt.Printf("size: %d bytes\n", tx.Size())
		fmt.Println("")
		return nil
	})
}
Example #15
0
func printDBDebug(db *bolt.DB) {
	dbBucketName := "alphazero"
	dbBucketLogs := fmt.Sprintf("%s.logs", dbBucketName)
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(dbBucketLogs))
		c := b.Cursor()
		i := 10
		for k, v := c.Last(); k != nil; k, v = c.Prev() {
			log := LogEntry{}
			json.Unmarshal(v, &log)
			fmt.Printf("- %s\n", log.String())
			i--
			if i == 0 {
				break
			}
		}
		return nil
	})

	dbBucketStats := fmt.Sprintf("%s.stats", dbBucketName)
	db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte(dbBucketStats)).Cursor()
		prefix := []byte("stats.")
		for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
			fmt.Printf("key=%s, value=%s\n", k, v)
		}
		return nil
	})
}
Example #16
0
// QueryDivisionByDivisionID querys Division by taxid
func QueryDivisionByDivisionID(db *bolt.DB, bucket string, ids []string) ([]nodes.Division, error) {
	for _, id := range ids {
		if !reDigitals.MatchString(id) {
			return []nodes.Division{}, fmt.Errorf("non-digital division given: %s", id)
		}
	}
	divisions := make([]nodes.Division, len(ids))
	if len(ids) == 0 {
		return divisions, nil
	}
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if b == nil {
			return fmt.Errorf("database not exists: %s", bucket)
		}
		for i, id := range ids {
			s := string(b.Get([]byte(id)))
			if s == "" {
				divisions[i] = nodes.Division{}
				continue
			}
			division, err := nodes.DivisionFromJSON(s)
			if err != nil {
				return errors.New("failed to parse division record from database")
			}
			divisions[i] = division
		}
		return nil
	})
	return divisions, err
}
Example #17
0
// LoadAllNodes loads all nodes into memory
func LoadAllNodes(db *bolt.DB, bucket string) (map[string]nodes.Node, error) {
	nods := make(map[string]nodes.Node)

	ch := make(chan string, runtime.NumCPU())
	chDone := make(chan int)
	go func() {
		for s := range ch {
			node, err := nodes.NodeFromJSON(s)
			checkError(err)
			nods[node.TaxID] = node
		}
		chDone <- 1
	}()

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if b == nil {
			return fmt.Errorf("database not exists: %s", bucket)
		}

		b.ForEach(func(k, v []byte) error {
			ch <- string(v)
			return nil
		})

		return nil
	})
	close(ch)
	<-chDone
	return nods, err
}
Example #18
0
func readItems(db *bolt.DB) map[string]godragon.Item {

	itemm := make(map[string]godragon.Item)

	db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte("items")).Cursor()

		var buff bytes.Buffer

		for k, v := c.First(); k != nil; k, v = c.Next() {

			var item godragon.Item
			dec := gob.NewDecoder(&buff)
			buff.Write(v)
			err := dec.Decode(&item)
			if err != nil {
				log.Println(err)
				return err
			}
			itemm[string(k)] = item
			buff.Reset()
		}
		return nil
	})

	return itemm

}
Example #19
0
// Exists checks if the key exists inside the store
func (b *BoltDB) Exists(key string) (bool, error) {
	var (
		val []byte
		db  *bolt.DB
		err error
	)
	b.Lock()
	defer b.Unlock()

	if db, err = b.getDBhandle(); err != nil {
		return false, err
	}
	defer b.releaseDBhandle()

	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(b.boltBucket)
		if bucket == nil {
			return store.ErrKeyNotFound
		}

		val = bucket.Get([]byte(key))

		return nil
	})

	if len(val) == 0 {
		return false, err
	}
	return true, err
}
Example #20
0
func viewAll(db *bolt.DB) {
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Shows"))
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			show, err := ShowFromBytes(v)
			if err != nil {
				fmt.Println("Error loading show '%s'", k)
				continue
			}

			name := show.Name
			hasNext := ""
			isPaused := ""

			if len(name) > 15 {
				name = name[:15] + "..."
			}

			if _, ok := show.NextPointer(); ok {
				hasNext = "+"
			}

			if show.Paused {
				isPaused = "paused"
			}

			fmt.Printf("%-20s %s %s %s\n", name, show.Pointer, hasNext, isPaused)
		}

		return nil
	})
}
Example #21
0
// walk walks recursively the bolt database db, calling walkFn for each key it finds.
func (cmd *CompactCommand) walk(db *bolt.DB, walkFn walkFunc) error {
	return db.View(func(tx *bolt.Tx) error {
		return tx.ForEach(func(name []byte, b *bolt.Bucket) error {
			return cmd.walkBucket(b, nil, name, nil, b.Sequence(), walkFn)
		})
	})
}
Example #22
0
func view(db *bolt.DB, name string) {
	var (
		err  error
		show *Show
	)

	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Shows"))
		show, err = ShowFromBytes(b.Get([]byte(name)))
		return err
	})

	if show == nil {
		fmt.Printf("The specified show '%s' not found.\n", name)
		return
	}

	fmt.Printf("Name        : %s\n", show.Name)
	fmt.Printf("Query       : %s\n", show.Query)
	fmt.Printf("Min seeders : %d\n", show.SeedersMin)
	fmt.Printf("Prefer HQ   : %t\n", show.PreferHQ)
	fmt.Printf("Pointer     : %s\n", show.Pointer)
	fmt.Printf("Paused      : %t\n", show.Paused)

	seasons := show.Seasons.Slice()

	if len(seasons) > 0 {
		fmt.Println("")
		fmt.Println("Season  Episodes  Begin")

		for _, season := range seasons {
			fmt.Printf("%-7d %-9d %s\n", season.Number, season.EpisodeCount, season.Begin)
		}
	}
}
Example #23
0
func addUser(db *bolt.DB, email string, password string) bool {
	check := []byte("")
	hashedPass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		fmt.Println(err)
	}
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("UsersBucket"))
		check = b.Get([]byte(email))
		return nil
	})
	if check == nil {
		db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte("UsersBucket"))
			err := b.Put([]byte(email), []byte(hashedPass))
			return err
		})
		if err != nil {
			fmt.Println(err)
		}
		return true
	} else {
		return false
	}
}
Example #24
0
func GetStatistics(db *bolt.DB) (*Statistics, error) {
	var stat Statistics

	err := db.View(func(tx *bolt.Tx) error {
		var err error

		if bucket := tx.Bucket(StatisticsBucketName); bucket != nil {
			stat.CountriesCount, err = strconv.Atoi(
				string(bucket.Get([]byte("countries_count"))),
			)

			stat.CitiesCount, err = strconv.Atoi(
				string(bucket.Get([]byte("cities_count"))),
			)

			stat.CityNamesCount, err = strconv.Atoi(
				string(bucket.Get([]byte("city_names_count"))),
			)
		}

		if err == nil && stat.CityNamesCount > 0 {
			stat.Status = "ok"
		} else {
			stat.Status = "indexing"
		}

		return err
	})

	return &stat, err
}
Example #25
0
func GetSchemaFromDb(b *bolt.DB) Schema {

	// first version
	v := First
	err := b.View(func(t *bolt.Tx) error {
		b := t.Bucket(AboutBucket)
		if b == nil {
			return nil
		}

		v = VersionFromString(string(b.Get([]byte("about"))))
		return nil
	})
	if err != nil {
		return Schemas[0]
	}

	// find the oldest valid schema
	for _, s := range Schemas {

		// best case, exact match
		if s.Version == v {
			return s
		}

		// as soon as we find an older version, this is the one
		if !s.Version.Greater(v) {
			return s
		}
	}

	return Schemas[0]
}
Example #26
0
func executeSelect(stmt *boltq.SelectStatement, db *bolt.DB) error {
	return db.View(func(tx *bolt.Tx) error {
		var bucket *bolt.Bucket

		for _, name := range stmt.BucketPath {
			log.Debugln("navigating to bucket", name)
			bucket = tx.Bucket([]byte(name))

			if bucket == nil {
				return fmt.Errorf("cannot find bucket %s", name)
			}
		}

		if containsAsterisk(stmt.Fields) {
			log.Debugln("interating keys")
			cursor := bucket.Cursor()

			for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
				emitKeypair(k, v)
			}
		} else {
			for _, k := range stmt.Fields {
				keyBytes := []byte(k)
				v := bucket.Get(keyBytes)
				emitKeypair(keyBytes, v)
			}
		}

		return nil
	})
}
Example #27
0
func PrintDb(db *bolt.DB, buckets []string) error {
	err := db.View(func(tx *bolt.Tx) error {
		// err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// 	fmt.Println(string(name))
		// 	return nil
		// })
		for _, bucket := range buckets {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				log.Printf("Bucket %v not found.", bucket)
				continue
			}
			fmt.Println(bucket, b.Stats().KeyN)
			err := b.ForEach(func(k, v []byte) error {
				fmt.Printf("\t%v -> %v\n", string(k), string(v))
				err := goInside(tx, v, 2)
				if err != nil {
					return e.Forward(err)
				}
				return nil
			})
			if err != nil {
				return e.Forward(err)
			}
		}
		return nil
	})
	if err != nil {
		return e.Forward(err)
	}
	return nil
}
Example #28
0
func getTotalItemFromDB(db *bolt.DB) (int64, error) {
	total_item := int64(0)
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(BUCKET_STATS))
		if b == nil {
			return nil
		}
		v := b.Get([]byte(KEY_TOTALITEM))
		if v == nil {
			return nil
		}
		tmp, err := strconv.ParseInt(string(v), 10, 64)
		if err != nil {
			// log.Printf("ERROR: parse count %v\n", err)
			return err
		}

		total_item = tmp
		return nil
	})
	if err != nil {
		return 0, err
	}
	return total_item, nil
}
Example #29
0
func readMeasurePoint(db *bolt.DB, bucketName string, name string) (MeasurePoint, error) {

	outMeasurePoint := MeasurePoint{}
	err := db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		// if no bucket - return nil (outMeasurePoint IS empty)
		if bucket == nil {
			return nil
		}

		valBuf := bucket.Get([]byte(name))
		// if no key - return nil (outMeasurePoint IS empty)
		if len(valBuf) == 0 {
			return nil
		}

		err := json.Unmarshal(valBuf, &outMeasurePoint)
		if err != nil {
			log.WithFields(log.Fields{
				"in":    "readMeasurePoint",
				"after": "Unmarshal",
				"ctx":   "Read from Bolt",
			}).Errorf("%s", err)
			Stat.ErrorIncr()
			return err
		}
		return nil
	})

	return outMeasurePoint, err
}
Example #30
0
func allLists(db *bolt.DB, params martini.Params, req *http.Request, r render.Render) {
	var lists []DistributionList

	err := db.View(
		func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucketDistributionLists))

			return b.ForEach(func(k, v []byte) error {
				var d DistributionList
				err := json.Unmarshal(v, &d)
				// TODO(dgryski): skip invalid keys instead of aborting on corrupt db?[
				if err != nil {
					return err
				}
				lists = append(lists, d)
				return nil
			})
		})

	if err != nil {
		r.Error(http.StatusInternalServerError)
		return
	}

	r.JSON(http.StatusOK, lists)
}