// Needle get a needle from sync.Pool.
func (s *Store) Needle() *needle.Needle {
var n interface{}
if n = s.np.Get(); n != nil {
return n.(*needle.Needle)
}
return needle.NewBufferNeedle(s.conf.NeedleMaxSize)
}
func TestVolume(t *testing.T) {
var (
v *Volume
err error
data = []byte("test")
bfile = "../test/test1"
ifile = "../test/test1.idx"
n = needle.NewBufferNeedle(_c.NeedleMaxSize)
ns = needle.NewBufferNeedles(3, _c.NeedleMaxSize)
buf = &bytes.Buffer{}
)
os.Remove(bfile)
os.Remove(ifile)
defer os.Remove(bfile)
defer os.Remove(ifile)
if v, err = NewVolume(1, bfile, ifile, _c); err != nil {
t.Errorf("NewVolume() error(%v)", err)
t.FailNow()
}
v.Close()
// test open
if err = v.Open(); err != nil {
t.Errorf("Open() error(%v)", err)
t.FailNow()
}
defer v.Close()
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(1, 1, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = v.Write(n); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(2, 2, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = v.Write(n); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(3, 3, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = v.Write(n); err != nil {
t.Errorf("Add() error(%v)", err)
t.FailNow()
}
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = ns.WriteFrom(4, 4, 4, buf); err != nil {
t.Errorf("ns.Write() error(%v)", err)
t.FailNow()
}
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = ns.WriteFrom(5, 5, 4, buf); err != nil {
t.Errorf("ns.Write() error(%v)", err)
t.FailNow()
}
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = ns.WriteFrom(6, 6, 4, buf); err != nil {
t.Errorf("ns.Write() error(%v)", err)
t.FailNow()
}
if err = v.Writes(ns); err != nil {
t.Errorf("Write() error(%v)", err)
t.FailNow()
}
if err = v.Delete(3); err != nil {
t.Errorf("Del error(%v)", err)
t.FailNow()
}
n.Key = 3
n.Cookie = 3
if err = v.Get(n); err != errors.ErrNeedleDeleted {
t.Error("err must be ErrNeedleDeleted")
t.FailNow()
} else {
err = nil
}
}
func TestStore(t *testing.T) {
var (
s *Store
z *zk.Zookeeper
v *volume.Volume
err error
n = needle.NewBufferNeedle(40)
buf = &bytes.Buffer{}
)
os.Remove(testConf.Store.VolumeIndex)
os.Remove(testConf.Store.FreeVolumeIndex)
os.Remove("./test/_free_block_1")
os.Remove("./test/_free_block_1.idx")
os.Remove("./test/_free_block_2")
os.Remove("./test/_free_block_2.idx")
os.Remove("./test/_free_block_3")
os.Remove("./test/_free_block_3.idx")
os.Remove("./test/1_0")
os.Remove("./test/1_0.idx")
os.Remove("./test/1_1")
os.Remove("./test/1_1.idx")
os.Remove("./test/block_store_1")
os.Remove("./test/block_store_1.idx")
defer os.Remove(testConf.Store.VolumeIndex)
defer os.Remove(testConf.Store.FreeVolumeIndex)
defer os.Remove("./test/_free_block_1")
defer os.Remove("./test/_free_block_1.idx")
defer os.Remove("./test/_free_block_2")
defer os.Remove("./test/_free_block_2.idx")
defer os.Remove("./test/_free_block_3")
defer os.Remove("./test/_free_block_3.idx")
defer os.Remove("./test/1_0")
defer os.Remove("./test/1_0.idx")
defer os.Remove("./test/1_1")
defer os.Remove("./test/1_1.idx")
defer os.Remove("./test/block_store_1")
defer os.Remove("./test/block_store_1.idx")
if z, err = zk.NewZookeeper(testConf); err != nil {
t.Errorf("NewZookeeper() error(%v)", err)
t.FailNow()
}
defer z.Close()
z.DelVolume(1)
z.DelVolume(2)
z.DelVolume(3)
defer z.DelVolume(1)
defer z.DelVolume(2)
defer z.DelVolume(3)
if s, err = NewStore(testConf); err != nil {
t.Errorf("NewStore() error(%v)", err)
t.FailNow()
}
defer s.Close()
if _, err = s.AddFreeVolume(2, "./test", "./test"); err != nil {
t.Errorf("s.AddFreeVolume() error(%v)", err)
t.FailNow()
}
if v, err = s.AddVolume(1); err != nil {
t.Errorf("AddVolume() error(%v)", err)
t.FailNow()
}
if v = s.Volumes[1]; v == nil {
t.Error("Volume(1) not exist")
t.FailNow()
}
buf.WriteString("test")
n.WriteFrom(1, 1, 4, buf)
if err = v.Write(n); err != nil {
t.Errorf("v.Add(1) error(%v)", err)
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
if err = s.BulkVolume(2, "./test/block_store_1", "./test/block_store_1.idx"); err != nil {
t.Errorf("Bulk(1) error(%v)", err)
t.FailNow()
}
if v = s.Volumes[2]; v == nil {
t.Error("Volume(2) not exist")
t.FailNow()
}
if err = v.Write(n); err != nil {
t.Errorf("v.Add() error(%v)", err)
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
if err = s.CompactVolume(1); err != nil {
t.Errorf("Compress(1) error(%v)", err)
t.FailNow()
}
if v = s.Volumes[1]; v == nil {
t.Error("Volume(1) not exist")
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
s.DelVolume(1)
if v = s.Volumes[1]; v != nil {
t.Error(err)
t.FailNow()
}
}
func TestSuperBlock(t *testing.T) {
var (
b *SuperBlock
offset, v2, v3, v4 uint32
err error
buf = &bytes.Buffer{}
needles = make(map[int64]int64)
data = []byte("test")
n = needle.NewBufferNeedle(4)
file = "../test/test.block"
ifile = "../test/test.idx"
//indexer *Indexer
)
os.Remove(file)
os.Remove(ifile)
defer os.Remove(file)
defer os.Remove(ifile)
// test new block file
if b, err = NewSuperBlock(file, testConf); err != nil {
t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
t.FailNow()
}
b.Close()
// test parse block file
if b, err = NewSuperBlock(file, testConf); err != nil {
t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
t.FailNow()
}
b.Close()
// test open
if err = b.Open(); err != nil {
t.Errorf("Open() error(%v)", err)
t.FailNow()
}
defer b.Close()
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(1, 1, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
if err = compareTestOffset(b, n, needle.NeedleOffset(int64(_headerSize))); err != nil {
t.Errorf("compareTestOffset() error(%v)", err)
t.FailNow()
}
offset = b.Offset
v2 = b.Offset
// test get
if err = b.ReadAt(1, n); err != nil {
t.Errorf("b.ReadAt() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 1, 1, needle.FlagOK, n, data); err != nil {
t.Errorf("compareTestNeedle() error(%v)", err)
t.FailNow()
}
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(2, 2, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
if err = compareTestOffset(b, n, offset); err != nil {
t.Errorf("compareTestOffset() error(%v)", err)
t.FailNow()
}
offset = b.Offset
v3 = b.Offset
if err = b.ReadAt(6, n); err != nil {
t.Errorf("b.ReadAt() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 2, 2, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(2)")
t.FailNow()
}
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(3, 3, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
offset = b.Offset
v4 = b.Offset
// test write
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(4, 4, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.Write(n); err != nil {
t.Errorf("b.Write() error(%v)", err)
t.FailNow()
}
if err = b.flush(true); err != nil {
t.Errorf("Flush() error(%v)", err)
t.FailNow()
}
if err = compareTestOffset(b, n, offset); err != nil {
t.Errorf("compareTestOffset() error(%v)", err)
t.FailNow()
}
if err = b.ReadAt(11, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(3)")
t.FailNow()
}
if err = b.ReadAt(16, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(r)")
t.FailNow()
}
// test del, del first needles
if err = b.Delete(1); err != nil {
t.Errorf("Del() error(%v)", err)
t.FailNow()
}
// test get
if err = b.ReadAt(1, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 1, 1, needle.FlagDel, n, data); err != nil {
t.FailNow()
}
if err = b.ReadAt(11, n); err != nil {
t.Errorf("Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
t.FailNow()
}
if err = b.ReadAt(16, n); err != nil {
t.Errorf("b.Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data); err != nil {
t.FailNow()
}
// test recovery
offset = b.Offset
if err = b.Recovery(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
if rn.Flag != needle.FlagOK {
so = needle.CacheDelOffset
}
needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
return
}); err != nil {
t.Errorf("Recovery() error(%v)", err)
t.FailNow()
}
if b.Offset != offset {
err = fmt.Errorf("b.Offset not match %d", b.Offset)
t.Error(err)
t.FailNow()
}
if o, s := needle.Cache(needles[1]); o != needle.CacheDelOffset && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
t.Error("needle.Cache() not match")
t.FailNow()
}
needles = make(map[int64]int64)
if err = b.Recovery(v2, func(rn *needle.Needle, so, eo uint32) (err1 error) {
if rn.Flag != needle.FlagOK {
so = needle.CacheDelOffset
}
needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
return
}); err != nil {
t.Errorf("b.Recovery() error(%v)", err)
t.FailNow()
}
// skip first needle, so key:1 must not exist
if o, s := needle.Cache(needles[1]); o != 0 && s != 0 {
t.Error("needle.Value(1) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
t.Error("needle.Value(2) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
t.Error("needle.Value(3) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
t.Error("needle.Value(4) not match")
t.FailNow()
}
// test repair
if _, err = buf.Write(data); err != nil {
t.Errorf("buf.Write() error(%v)", err)
t.FailNow()
}
if err = n.WriteFrom(3, 3, 4, buf); err != nil {
t.Errorf("n.Write() error(%v)", err)
t.FailNow()
}
if err = b.WriteAt(v3, n); err != nil {
t.Errorf("b.Repair(3) error(%v)", err)
t.FailNow()
}
if err = b.ReadAt(v3, n); err != nil {
t.Errorf("b.Get() error(%v)", err)
t.FailNow()
}
if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
t.Error("compareTestNeedle(3)")
t.FailNow()
}
// test compress
if err = b.Compact(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
if rn.Flag != needle.FlagOK {
return
}
needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
return
}); err != nil {
t.Errorf("b.Compress() error(%v)", err)
t.FailNow()
}
// skip first needle, so key:1 must not exist
if o, s := needle.Cache(needles[1]); o != 0 && s != 0 {
t.Error("needle.Value(1) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
t.Error("needle.Value(2) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
t.Error("needle.Value(3) not match")
t.FailNow()
}
if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
t.Error("needle.Value(4) not match")
t.FailNow()
}
}