Example #1
0
func writeBatch(db storage.Engine, c *Config) int {
	ws := c.MakeBatch()
	if err := db.BatchPut(ws); err != nil {
		panic(err)
	}
	return len(ws)
}
Example #2
0
func queryAndDelete(db storage.Engine, points, series int) {
	// query the database
	startCount := points / series / 4
	endCount := points * 3 / series / 4

	total := 0
	var d time.Duration

	for series -= 1; series >= 0; series-- {
		count := 0
		var delStart []byte
		var delEnd []byte
		query(db, int64(series), func(itr storage.Iterator) {
			count++
			if count == startCount {
				delStart = itr.Key()
			}
			if count == endCount-1 {
				delEnd = itr.Key()
				total += endCount - startCount
			}
		})

		start := time.Now()
		err := db.Del(delStart, delEnd)
		if err != nil {
			panic(err)
		}
		d += time.Now().Sub(start)
	}
	fmt.Printf("Took %s to delete %d points\n", d, total)
	start := time.Now()
	db.Compact()
	fmt.Printf("Took %s to compact\n", time.Now().Sub(start))
}
Example #3
0
func NewShard(db storage.Engine, pointBatchSize, writeBatchSize int) (*Shard, error) {
	lastIdBytes, err2 := db.Get(NEXT_ID_KEY)
	if err2 != nil {
		return nil, err2
	}

	lastId := uint64(0)
	if lastIdBytes != nil {
		lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes))
		if err2 != nil {
			return nil, err2
		}
	}

	return &Shard{
		db:             db,
		lastIdUsed:     lastId,
		pointBatchSize: pointBatchSize,
		writeBatchSize: writeBatchSize,
	}, nil
}
Example #4
0
func query(db storage.Engine, s int64, yield func(storage.Iterator)) {
	sb := bytes.NewBuffer(nil)
	binary.Write(sb, binary.BigEndian, s)
	binary.Write(sb, binary.BigEndian, int64(0))
	binary.Write(sb, binary.BigEndian, int64(0))
	eb := bytes.NewBuffer(nil)
	binary.Write(eb, binary.BigEndian, s)
	binary.Write(eb, binary.BigEndian, int64(-1))
	binary.Write(eb, binary.BigEndian, int64(-1))

	itr := db.Iterator()
	defer itr.Close()
	count := 0
	for itr.Seek(sb.Bytes()); itr.Valid(); itr.Next() {
		key := itr.Key()
		if bytes.Compare(key, eb.Bytes()) > 0 {
			break
		}

		count++
		yield(itr)
	}
}
Example #5
0
func benchmarkDbCommon(db storage.Engine, c Config) {
	fmt.Printf("################ Benchmarking: %s\n", db.Name())
	start := time.Now()
	count := 0
	for p := c.points; p > 0; {
		c := writeBatch(db, &c)
		count += c
		p -= c
	}
	d := time.Now().Sub(start)

	fmt.Printf("Writing %d points in batches of %d points took %s (%f microsecond per point)\n",
		count,
		c.batch,
		d,
		float64(d.Nanoseconds())/1000.0/float64(count),
	)

	timeQuerying(db, c.series)
	fmt.Printf("Size: %s\n", getSize(db.Path()))
	queryAndDelete(db, c.points, c.series)
	timeQuerying(db, c.series)
	fmt.Printf("Size: %s\n", getSize(db.Path()))

	start = time.Now()
	count = 0
	for p := c.points / 2; p > 0; {
		c := writeBatch(db, &c)
		count += c
		p -= c
	}
	d = time.Now().Sub(start)
	fmt.Printf("Writing %d points in batches of %d points took %s (%f microsecond per point)\n",
		count,
		c.batch,
		d,
		float64(d.Nanoseconds())/1000.0/float64(count),
	)
	fmt.Printf("Size: %s\n", getSize(db.Path()))
}