Esempio n. 1
0
func TestSuperBlock(t *testing.T) {
	var (
		b                  *SuperBlock
		n                  *needle.Needle
		offset, v2, v3, v4 uint32
		err                error
		buf                = &bytes.Buffer{}
		needles            = make(map[int64]int64)
		data               = []byte("test")
		file               = "../test/test.block"
		ifile              = "../test/test.idx"
		//indexer *Indexer
	)
	os.Remove(file)
	os.Remove(ifile)
	defer os.Remove(file)
	defer os.Remove(ifile)
	// test new block file
	if b, err = NewSuperBlock(file, testConf); err != nil {
		t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
		t.FailNow()
	}
	b.Close()
	// test parse block file
	if b, err = NewSuperBlock(file, testConf); err != nil {
		t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
		t.FailNow()
	}
	b.Close()
	// test open
	if err = b.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer b.Close()
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(1, 1, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, needle.NeedleOffset(int64(_headerSize))); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v2 = b.Offset
	// test get
	n.Offset = 1
	if err = b.ReadAt(n); err != nil {
		t.Errorf("b.ReadAt() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 1, 1, needle.FlagOK, n, data); err != nil {
		t.Errorf("compareTestNeedle() error(%v)", err)
		t.FailNow()
	}
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(2, 2, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, offset); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v3 = b.Offset
	n.Offset = 6
	if err = b.ReadAt(n); err != nil {
		t.Errorf("b.ReadAt() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 2, 2, needle.FlagOK, n, data); err != nil {
		t.Error("compareTestNeedle(2)")
		t.FailNow()
	}
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(3, 3, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v4 = b.Offset
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(4, 4, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
		t.FailNow()
	}
	if err = b.flush(true); err != nil {
		t.Errorf("Flush() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, offset); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	n.Offset = 11
	if err = b.ReadAt(n); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
		t.Error("compareTestNeedle(3)")
		t.FailNow()
	}
	n.Offset = 16
	if err = b.ReadAt(n); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data); err != nil {
		t.Error("compareTestNeedle(r)")
		t.FailNow()
	}
	// test del, del first needles
	if err = b.Delete(1); err != nil {
		t.Errorf("Del() error(%v)", err)
		t.FailNow()
	}
	// test get
	n.Offset = 1
	if err = b.ReadAt(n); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 1, 1, needle.FlagDel, n, data); err != nil {
		t.FailNow()
	}
	n.Offset = 11
	if err = b.ReadAt(n); err != nil {
		t.Errorf("Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
		t.FailNow()
	}
	n.Offset = 16
	if err = b.ReadAt(n); err != nil {
		t.Errorf("b.Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 4, 4, needle.FlagOK, n, data); err != nil {
		t.FailNow()
	}
	// test recovery
	offset = b.Offset
	if err = b.Recovery(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
		if rn.Flag != needle.FlagOK {
			so = needle.CacheDelOffset
		}
		needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
		return
	}); err != nil {
		t.Errorf("Recovery() error(%v)", err)
		t.FailNow()
	}
	if b.Offset != offset {
		err = fmt.Errorf("b.Offset not match %d", b.Offset)
		t.Error(err)
		t.FailNow()
	}
	if o, s := needle.Cache(needles[1]); o != needle.CacheDelOffset && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
		t.Error("needle.Cache() not match")
		t.FailNow()
	}
	needles = make(map[int64]int64)
	if err = b.Recovery(v2, func(rn *needle.Needle, so, eo uint32) (err1 error) {
		if rn.Flag != needle.FlagOK {
			so = needle.CacheDelOffset
		}
		needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
		return
	}); err != nil {
		t.Errorf("b.Recovery() error(%v)", err)
		t.FailNow()
	}
	// skip first needle, so key:1 must not exist
	if o, s := needle.Cache(needles[1]); o != 0 && s != 0 {
		t.Error("needle.Value(1) not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
		t.Error("needle.Value(2) not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
		t.Error("needle.Value(3) not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
		t.Error("needle.Value(4) not match")
		t.FailNow()
	}
	// test repair
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(3, 3, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.WriteAt(v3, n); err != nil {
		t.Errorf("b.Repair(3) error(%v)", err)
		t.FailNow()
	}
	n.Offset = v3
	if err = b.ReadAt(n); err != nil {
		t.Errorf("b.Get() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 3, 3, needle.FlagOK, n, data); err != nil {
		t.Error("compareTestNeedle(3)")
		t.FailNow()
	}
	// test compress
	if err = b.Compact(0, func(rn *needle.Needle, so, eo uint32) (err1 error) {
		if rn.Flag != needle.FlagOK {
			return
		}
		needles[rn.Key] = needle.NewCache(so, rn.TotalSize)
		return
	}); err != nil {
		t.Errorf("b.Compress() error(%v)", err)
		t.FailNow()
	}
	// skip first needle, so key:1 must not exist
	if o, s := needle.Cache(needles[1]); o != 0 && s != 0 {
		t.Error("needle.Value(1) not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[2]); o != v2 && s != 40 {
		t.Error("needle.Value(2) not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[3]); o != v3 && s != 40 {
		t.Error("needle.Value(3) not match")
		t.FailNow()
	}
	if o, s := needle.Cache(needles[4]); o != v4 && s != 40 {
		t.Error("needle.Value(4) not match")
		t.FailNow()
	}
}
Esempio n. 2
0
func TestVolume(t *testing.T) {
	var (
		v     *Volume
		n     *needle.Needle
		err   error
		data  = []byte("test")
		bfile = "../test/test1"
		ifile = "../test/test1.idx"
		ns    = needle.NewNeedles(3)
		buf   = &bytes.Buffer{}
	)
	os.Remove(bfile)
	os.Remove(ifile)
	defer os.Remove(bfile)
	defer os.Remove(ifile)
	if v, err = NewVolume(1, bfile, ifile, _c); err != nil {
		t.Errorf("NewVolume() error(%v)", err)
		t.FailNow()
	}
	v.Close()
	// test open
	if err = v.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer v.Close()
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(1, 1, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(2, 2, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(3, 3, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	if err = ns.ReadFrom(4, 4, 4, buf); err != nil {
		t.Errorf("ns.Write() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	if err = ns.ReadFrom(5, 5, 4, buf); err != nil {
		t.Errorf("ns.Write() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	if err = ns.ReadFrom(6, 6, 4, buf); err != nil {
		t.Errorf("ns.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Writes(ns); err != nil {
		t.Errorf("Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Delete(3); err != nil {
		t.Errorf("Del error(%v)", err)
		t.FailNow()
	}
	if _, err = v.Read(3, 3); err != errors.ErrNeedleDeleted {
		t.Error("err must be ErrNeedleDeleted")
		t.FailNow()
	} else {
		err = nil
	}
}
Esempio n. 3
0
func TestStore(t *testing.T) {
	var (
		s   *Store
		z   *zk.Zookeeper
		v   *volume.Volume
		n   *needle.Needle
		err error
		buf = &bytes.Buffer{}
	)
	os.Remove(testConf.Store.VolumeIndex)
	os.Remove(testConf.Store.FreeVolumeIndex)
	os.Remove("./test/_free_block_1")
	os.Remove("./test/_free_block_1.idx")
	os.Remove("./test/_free_block_2")
	os.Remove("./test/_free_block_2.idx")
	os.Remove("./test/_free_block_3")
	os.Remove("./test/_free_block_3.idx")
	os.Remove("./test/1_0")
	os.Remove("./test/1_0.idx")
	os.Remove("./test/1_1")
	os.Remove("./test/1_1.idx")
	os.Remove("./test/block_store_1")
	os.Remove("./test/block_store_1.idx")
	defer os.Remove(testConf.Store.VolumeIndex)
	defer os.Remove(testConf.Store.FreeVolumeIndex)
	defer os.Remove("./test/_free_block_1")
	defer os.Remove("./test/_free_block_1.idx")
	defer os.Remove("./test/_free_block_2")
	defer os.Remove("./test/_free_block_2.idx")
	defer os.Remove("./test/_free_block_3")
	defer os.Remove("./test/_free_block_3.idx")
	defer os.Remove("./test/1_0")
	defer os.Remove("./test/1_0.idx")
	defer os.Remove("./test/1_1")
	defer os.Remove("./test/1_1.idx")
	defer os.Remove("./test/block_store_1")
	defer os.Remove("./test/block_store_1.idx")
	if z, err = zk.NewZookeeper(testConf); err != nil {
		t.Errorf("NewZookeeper() error(%v)", err)
		t.FailNow()
	}
	defer z.Close()
	z.DelVolume(1)
	z.DelVolume(2)
	z.DelVolume(3)
	defer z.DelVolume(1)
	defer z.DelVolume(2)
	defer z.DelVolume(3)
	if s, err = NewStore(testConf); err != nil {
		t.Errorf("NewStore() error(%v)", err)
		t.FailNow()
	}
	defer s.Close()
	if _, err = s.AddFreeVolume(2, "./test", "./test"); err != nil {
		t.Errorf("s.AddFreeVolume() error(%v)", err)
		t.FailNow()
	}
	if v, err = s.AddVolume(1); err != nil {
		t.Errorf("AddVolume() error(%v)", err)
		t.FailNow()
	}
	if v = s.Volumes[1]; v == nil {
		t.Error("Volume(1) not exist")
		t.FailNow()
	}
	buf.WriteString("test")
	n = needle.NewWriter(1, 1, 4)
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.ReadFrom() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("v.Add(1) error(%v)", err)
		t.FailNow()
	}
	if _, err = v.Read(1, 1); err != nil {
		t.Errorf("v.WriteTo(1) error(%v)", err)
		t.FailNow()
	}
	if err = s.BulkVolume(2, "./test/block_store_1", "./test/block_store_1.idx"); err != nil {
		t.Errorf("Bulk(1) error(%v)", err)
		t.FailNow()
	}
	if v = s.Volumes[2]; v == nil {
		t.Error("Volume(2) not exist")
		t.FailNow()
	}
	buf.WriteString("test")
	n = needle.NewWriter(1, 1, 4)
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.ReadFrom() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("v.Add() error(%v)", err)
		t.FailNow()
	}
	if n, err = v.Read(1, 1); err != nil {
		t.Errorf("v.WriteTo(1) error(%v)", err)
		t.FailNow()
	} else {
		n.Close()
	}
	if err = s.CompactVolume(1); err != nil {
		t.Errorf("Compress(1) error(%v)", err)
		t.FailNow()
	}
	if v = s.Volumes[1]; v == nil {
		t.Error("Volume(1) not exist")
		t.FailNow()
	}
	if n, err = v.Read(1, 1); err != nil {
		t.Errorf("v.WriteTo(1) error(%v)", err)
		t.FailNow()
	} else {
		n.Close()
	}
	s.DelVolume(1)
	if v = s.Volumes[1]; v != nil {
		t.Error(err)
		t.FailNow()
	}
}