func TestBitCask(t *testing.T) {
// clear dirty
os.RemoveAll("testBitcask")
b, err := Open("testBitcask", nil)
logger.Info(err)
assert.Nil(t, err)
assert.NotNil(t, b)
testKey := []byte("Foo")
value := []byte("Bar")
b.Put(testKey, value)
v, err := b.Get(testKey)
assert.Nil(t, err)
logger.Info("value:", string(v))
assert.Equal(t, v, value)
testKey = []byte("xiaoMing")
value = []byte("abc")
b.Put(testKey, value)
v, err = b.Get(testKey)
logger.Info("value:", string(v))
assert.Equal(t, v, value)
// hintFile:
value = []byte("ddddd")
b.Put(testKey, value)
v, err = b.Get(testKey)
logger.Info("value:", string(v))
assert.Equal(t, v, value)
b.Close()
}
func BenchmarkBitcaskCurrency(b *testing.B) {
storagePath := "benchMarkBitcask"
os.RemoveAll(storagePath)
opts := &Options{
MaxFileSize: 1 << 12,
}
bc, err := Open(storagePath, opts)
if err != nil {
logger.Fatal(err)
}
keyValues := make(map[int]string)
for i := 0; i < b.N/2; i++ {
key := strconv.Itoa(i)
value := strconv.Itoa(int(time.Now().Unix()))
bc.Put([]byte(key), []byte(value))
keyValues[i] = value
}
logger.Warn(b.N)
logger.Info("Put all Data")
for i := 0; i < b.N/2; i++ {
k := strconv.Itoa(i)
v, _ := bc.Get([]byte(k))
if string(v) != keyValues[i] {
logger.Error(string(v), keyValues[i])
os.Exit(-1)
}
}
logger.Info("Get all data")
// delete all data
for i := 0; i < b.N/2; i++ {
k := strconv.Itoa(i)
//v, _ := bc.Get([]byte(k))
err := bc.Del([]byte(k))
if err != nil {
logger.Error(err)
}
}
logger.Info("Delete all data")
// Get all data
for i := 0; i < b.N/2; i++ {
k := strconv.Itoa(i)
v, err := bc.Get([]byte(k))
if err != ErrNotFound {
logger.Error(string(v), keyValues[i])
}
}
logger.Info("all data is not found, pass test")
//mergeWorker.Staop()
bc.Close()
}
func TestSplit2(t *testing.T) {
storagePath := "split2Bitcask"
os.RemoveAll(storagePath)
opts := &Options{
MaxFileSize: 2,
}
bc, err := Open(storagePath, opts)
assert.Nil(t, err)
testKey := []byte("Foo")
value := []byte("Bar")
bc.Put(testKey, value)
v, err := bc.Get(testKey)
assert.Nil(t, err)
assert.Equal(t, v, value)
logger.Info("==============================")
time.Sleep(time.Second * 2)
// cause split file
value = []byte("Apple")
bc.Put(testKey, value)
v, err = bc.Get(testKey)
assert.Nil(t, err)
assert.Equal(t, v, value)
bc.Close()
}
func main() {
flag.StringVar(&addr, "addr", "127.0.0.1:80", "bitcask http listen addr")
flag.StringVar(&storagePath, "s", "bitcaskStorage", "data storage path")
flag.Uint64Var(&maxSize, "ms", 1<<32, "single data file maxsize")
flag.IntVar(&logLevel, "l", 0, "logger level")
flag.Parse()
logger.SetLevel(1)
opts := &bitcask.Options{
MaxFileSize: maxSize,
}
var err error
bc, err = bitcask.Open(storagePath, opts)
if err != nil {
logger.Fatal(err)
}
defer bc.Close()
defer func() {
if err := recover(); err != nil {
logger.Error(err)
debug.PrintStack()
}
}()
r := mux.NewRouter()
r.HandleFunc("/{key}", bitcaskGetHandle).Methods("GET")
r.HandleFunc("/{key}", bitcaskDelHandle).Methods("DELETE")
r.HandleFunc("/{key}", bitcaskPutHandle).Methods("POST")
logger.Info("bitcask server listen:", addr)
if err := http.ListenAndServe(addr, r); err != nil {
logger.Error(err)
}
}
// DecodeEntry ...
func DecodeEntry(buf []byte) ([]byte, error) {
/**
crc32 : tStamp : ksz : valueSz : key : value
4 : 4 : 4 : 4 : xxxx : xxxx
**/
ksz := binary.LittleEndian.Uint32(buf[8:12])
valuesz := binary.LittleEndian.Uint32(buf[12:HeaderSize])
c32 := binary.LittleEndian.Uint32(buf[:4])
value := make([]byte, valuesz)
copy(value, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)])
logger.Info(c32)
if crc32.ChecksumIEEE(buf[4:]) != c32 {
return nil, ErrCrc32
}
return value, nil
}
func main() {
os.RemoveAll("exampleBitcaskDir")
bc, err := bitcask.Open("exampleBitcaskDir", nil)
if err != nil {
logger.Fatal(err)
}
defer bc.Close()
k1 := []byte("xiaoMing")
v1 := []byte("毕业于新东方推土机学院")
k2 := []byte("zhanSan")
v2 := []byte("毕业于新东方厨师学院")
bc.Put(k1, v1)
bc.Put(k2, v2)
v1, _ = bc.Get(k1)
v2, _ = bc.Get(k2)
logger.Info(string(k1), string(v1))
logger.Info(string(k2), string(v2))
// override
v2 = []byte("毕业于新东方美容美发学院")
bc.Put(k2, v2)
v2, _ = bc.Get(k2)
logger.Info(string(k2), string(v2))
bc.Del(k1)
bc.Del(k2)
logger.Info("毕业后的数据库：")
v1, e := bc.Get(k1)
if e != bitcask.ErrNotFound {
logger.Info(string(k1), "shoud be:", bitcask.ErrNotFound)
} else {
logger.Info(string(k1), "已经毕业.")
}
v2, e = bc.Get(k2)
if e != bitcask.ErrNotFound {
logger.Info(string(k1), "shoud be:", bitcask.ErrNotFound)
} else {
logger.Info(string(k2), "已经毕业.")
}
}
func d1() {
buf := make([]byte, bitcask.HeaderSize)
fp, err := os.Open(os.Args[1])
if err != nil {
logger.Fatal(err)
}
offset := int64(0)
for {
n, err := fp.ReadAt(buf, offset)
if err != nil && err != io.EOF {
logger.Fatal(err)
}
if err == io.EOF {
break
}
if n != len(buf) || n != bitcask.HeaderSize {
logger.Fatal(n)
}
offset += int64(n)
// parse data header
c32, tStamp, ksz, valuesz := bitcask.DecodeEntryHeader(buf)
logger.Info(c32, tStamp, "ksz:", ksz, "valuesz:", valuesz)
if err != nil {
logger.Fatal(err)
}
if ksz+valuesz == 0 {
continue
}
keyValue := make([]byte, ksz+valuesz)
n, err = fp.ReadAt(keyValue, offset)
if err != nil && err != io.EOF {
logger.Fatal(err)
}
if err == io.EOF {
break
}
offset += int64(n)
fmt.Println(string(keyValue[:ksz]), string(keyValue[ksz:]))
}
}
// if writeableFile size large than Opts.MaxFileSize and the fileID not equal to local time stamp;
// if will create a new writeable file
func checkWriteableFile(bc *BitCask) {
if bc.writeFile.writeOffset > bc.Opts.MaxFileSize && bc.writeFile.fileID != uint32(time.Now().Unix()) {
logger.Info("open a new data/hint file:", bc.writeFile.writeOffset, bc.Opts.MaxFileSize)
//close data/hint fp
bc.writeFile.hintFp.Close()
bc.writeFile.fp.Close()
writeFp, fileID := setWriteableFile(0, bc.dirFile)
hintFp := setHintFile(fileID, bc.dirFile)
bf := &BFile{
fp: writeFp,
fileID: fileID,
writeOffset: 0,
hintFp: hintFp,
}
bc.writeFile = bf
// update pid
writePID(bc.lockFile, fileID)
}
}
func d2() {
buf := make([]byte, bitcask.HeaderSize)
fp, err := os.Open(os.Args[1])
if err != nil {
logger.Fatal(err)
}
for {
n, err := fp.Read(buf[0:])
if err != nil && err != io.EOF {
logger.Fatal(err)
}
if n != len(buf) {
logger.Fatal(n)
}
value, err := bitcask.DecodeEntry(buf)
logger.Info(value)
if err != nil {
logger.Fatal(err)
}
//logger.Info(c32, tStamp, ksz, valuesz, key, value)
}
}