예제 #1
0
func TestBitCask(t *testing.T) {
	// clear dirty
	os.RemoveAll("testBitcask")
	b, err := Open("testBitcask", nil)
	logger.Info(err)
	assert.Nil(t, err)
	assert.NotNil(t, b)

	testKey := []byte("Foo")
	value := []byte("Bar")
	b.Put(testKey, value)
	v, err := b.Get(testKey)
	assert.Nil(t, err)
	logger.Info("value:", string(v))
	assert.Equal(t, v, value)

	testKey = []byte("xiaoMing")
	value = []byte("abc")
	b.Put(testKey, value)
	v, err = b.Get(testKey)
	logger.Info("value:", string(v))
	assert.Equal(t, v, value)

	// hintFile:
	value = []byte("ddddd")
	b.Put(testKey, value)
	v, err = b.Get(testKey)
	logger.Info("value:", string(v))
	assert.Equal(t, v, value)

	b.Close()
}
예제 #2
0
func BenchmarkBitcaskCurrency(b *testing.B) {
	storagePath := "benchMarkBitcask"
	os.RemoveAll(storagePath)
	opts := &Options{
		MaxFileSize: 1 << 12,
	}
	bc, err := Open(storagePath, opts)
	if err != nil {
		logger.Fatal(err)
	}

	keyValues := make(map[int]string)

	for i := 0; i < b.N/2; i++ {
		key := strconv.Itoa(i)
		value := strconv.Itoa(int(time.Now().Unix()))
		bc.Put([]byte(key), []byte(value))
		keyValues[i] = value
	}
	logger.Warn(b.N)
	logger.Info("Put all Data")
	for i := 0; i < b.N/2; i++ {
		k := strconv.Itoa(i)
		v, _ := bc.Get([]byte(k))
		if string(v) != keyValues[i] {
			logger.Error(string(v), keyValues[i])
			os.Exit(-1)
		}
	}
	logger.Info("Get all data")
	// delete all data
	for i := 0; i < b.N/2; i++ {
		k := strconv.Itoa(i)
		//v, _ := bc.Get([]byte(k))
		err := bc.Del([]byte(k))
		if err != nil {
			logger.Error(err)
		}
	}
	logger.Info("Delete all data")
	// Get all data
	for i := 0; i < b.N/2; i++ {
		k := strconv.Itoa(i)
		v, err := bc.Get([]byte(k))
		if err != ErrNotFound {
			logger.Error(string(v), keyValues[i])
		}
	}
	logger.Info("all data is not found, pass test")
	//mergeWorker.Staop()
	bc.Close()
}
예제 #3
0
func TestSplit2(t *testing.T) {
	storagePath := "split2Bitcask"
	os.RemoveAll(storagePath)
	opts := &Options{
		MaxFileSize: 2,
	}
	bc, err := Open(storagePath, opts)
	assert.Nil(t, err)
	testKey := []byte("Foo")

	value := []byte("Bar")
	bc.Put(testKey, value)
	v, err := bc.Get(testKey)
	assert.Nil(t, err)
	assert.Equal(t, v, value)
	logger.Info("==============================")
	time.Sleep(time.Second * 2)

	// cause split file
	value = []byte("Apple")
	bc.Put(testKey, value)
	v, err = bc.Get(testKey)
	assert.Nil(t, err)
	assert.Equal(t, v, value)
	bc.Close()
}
예제 #4
0
func main() {
	flag.StringVar(&addr, "addr", "127.0.0.1:80", "bitcask http listen addr")
	flag.StringVar(&storagePath, "s", "bitcaskStorage", "data storage path")
	flag.Uint64Var(&maxSize, "ms", 1<<32, "single data file maxsize")
	flag.IntVar(&logLevel, "l", 0, "logger level")
	flag.Parse()

	logger.SetLevel(1)
	opts := &bitcask.Options{
		MaxFileSize: maxSize,
	}
	var err error
	bc, err = bitcask.Open(storagePath, opts)
	if err != nil {
		logger.Fatal(err)
	}
	defer bc.Close()

	defer func() {
		if err := recover(); err != nil {
			logger.Error(err)
			debug.PrintStack()
		}
	}()

	r := mux.NewRouter()
	r.HandleFunc("/{key}", bitcaskGetHandle).Methods("GET")
	r.HandleFunc("/{key}", bitcaskDelHandle).Methods("DELETE")
	r.HandleFunc("/{key}", bitcaskPutHandle).Methods("POST")
	logger.Info("bitcask server listen:", addr)
	if err := http.ListenAndServe(addr, r); err != nil {
		logger.Error(err)
	}
}
예제 #5
0
// DecodeEntry ...
func DecodeEntry(buf []byte) ([]byte, error) {
	/**
	    crc32	:	tStamp	:	ksz	:	valueSz	:	key	:	value
	    4 		:	4 		: 	4 	: 		4	:	xxxx	: xxxx
	**/
	ksz := binary.LittleEndian.Uint32(buf[8:12])

	valuesz := binary.LittleEndian.Uint32(buf[12:HeaderSize])
	c32 := binary.LittleEndian.Uint32(buf[:4])
	value := make([]byte, valuesz)
	copy(value, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)])
	logger.Info(c32)
	if crc32.ChecksumIEEE(buf[4:]) != c32 {
		return nil, ErrCrc32
	}
	return value, nil
}
예제 #6
0
func main() {
	os.RemoveAll("exampleBitcaskDir")
	bc, err := bitcask.Open("exampleBitcaskDir", nil)
	if err != nil {
		logger.Fatal(err)
	}
	defer bc.Close()
	k1 := []byte("xiaoMing")
	v1 := []byte("毕业于新东方推土机学院")

	k2 := []byte("zhanSan")
	v2 := []byte("毕业于新东方厨师学院")

	bc.Put(k1, v1)
	bc.Put(k2, v2)

	v1, _ = bc.Get(k1)
	v2, _ = bc.Get(k2)
	logger.Info(string(k1), string(v1))
	logger.Info(string(k2), string(v2))
	// override
	v2 = []byte("毕业于新东方美容美发学院")
	bc.Put(k2, v2)
	v2, _ = bc.Get(k2)
	logger.Info(string(k2), string(v2))

	bc.Del(k1)
	bc.Del(k2)
	logger.Info("毕业后的数据库:")
	v1, e := bc.Get(k1)
	if e != bitcask.ErrNotFound {
		logger.Info(string(k1), "shoud be:", bitcask.ErrNotFound)
	} else {
		logger.Info(string(k1), "已经毕业.")
	}
	v2, e = bc.Get(k2)
	if e != bitcask.ErrNotFound {
		logger.Info(string(k1), "shoud be:", bitcask.ErrNotFound)
	} else {
		logger.Info(string(k2), "已经毕业.")
	}

}
예제 #7
0
func d1() {
	buf := make([]byte, bitcask.HeaderSize)
	fp, err := os.Open(os.Args[1])
	if err != nil {
		logger.Fatal(err)
	}

	offset := int64(0)
	for {
		n, err := fp.ReadAt(buf, offset)
		if err != nil && err != io.EOF {
			logger.Fatal(err)
		}
		if err == io.EOF {
			break
		}
		if n != len(buf) || n != bitcask.HeaderSize {
			logger.Fatal(n)
		}
		offset += int64(n)
		// parse data header
		c32, tStamp, ksz, valuesz := bitcask.DecodeEntryHeader(buf)
		logger.Info(c32, tStamp, "ksz:", ksz, "valuesz:", valuesz)
		if err != nil {
			logger.Fatal(err)
		}

		if ksz+valuesz == 0 {
			continue
		}

		keyValue := make([]byte, ksz+valuesz)
		n, err = fp.ReadAt(keyValue, offset)
		if err != nil && err != io.EOF {
			logger.Fatal(err)
		}
		if err == io.EOF {
			break
		}
		offset += int64(n)
		fmt.Println(string(keyValue[:ksz]), string(keyValue[ksz:]))
	}
}
예제 #8
0
파일: util.go 프로젝트: laohanlinux/bitcask
// if writeableFile size large than Opts.MaxFileSize and the fileID not equal to local time stamp;
// if will create a new writeable file
func checkWriteableFile(bc *BitCask) {
	if bc.writeFile.writeOffset > bc.Opts.MaxFileSize && bc.writeFile.fileID != uint32(time.Now().Unix()) {
		logger.Info("open a new data/hint file:", bc.writeFile.writeOffset, bc.Opts.MaxFileSize)
		//close data/hint fp
		bc.writeFile.hintFp.Close()
		bc.writeFile.fp.Close()

		writeFp, fileID := setWriteableFile(0, bc.dirFile)
		hintFp := setHintFile(fileID, bc.dirFile)
		bf := &BFile{
			fp:          writeFp,
			fileID:      fileID,
			writeOffset: 0,
			hintFp:      hintFp,
		}
		bc.writeFile = bf
		// update pid
		writePID(bc.lockFile, fileID)
	}
}
예제 #9
0
func d2() {
	buf := make([]byte, bitcask.HeaderSize)
	fp, err := os.Open(os.Args[1])
	if err != nil {
		logger.Fatal(err)
	}

	for {
		n, err := fp.Read(buf[0:])
		if err != nil && err != io.EOF {
			logger.Fatal(err)
		}
		if n != len(buf) {
			logger.Fatal(n)
		}
		value, err := bitcask.DecodeEntry(buf)
		logger.Info(value)
		if err != nil {
			logger.Fatal(err)
		}
		//logger.Info(c32, tStamp, ksz, valuesz, key, value)
	}
}