func TestNewHeader(t *testing.T) {
h1 := new_header(4, 8, false)
h2 := new_header(4, 8, true)
b1 := h1.Bytes()
b2 := h2.Bytes()
h1_2, _ := load_header(b1)
h2_2, _ := load_header(b2)
if !bs.ByteSlice(h1_2.Bytes()).Eq(b1) {
t.Fatal("h1 != h1_2")
}
if !bs.ByteSlice(h2_2.Bytes()).Eq(b2) {
t.Fatal("h2 != h2_2")
}
o1 := h1_2.flags()
o2 := h2_2.flags()
if o1 {
t.Fatal("h1 flag not set properly")
}
if !o2 {
t.Fatal("h2 flag not set properly")
}
}
func load_ctrlblk(bytes []byte) (cb *ctrlblk, err error) {
if len(bytes) < CONTROLSIZE {
return nil, fmt.Errorf("len(bytes) < %d", CONTROLSIZE)
}
cb = &ctrlblk{
end: int64(bs.ByteSlice(bytes[0:8]).Int64()),
free_head: int64(bs.ByteSlice(bytes[8:16]).Int64()),
free_len: bs.ByteSlice(bytes[16:20]).Int32(),
}
return cb, nil
}
func init() {
if urandom, err := os.Open("/dev/urandom"); err != nil {
return
} else {
seed := make([]byte, 8)
if _, err := urandom.Read(seed); err == nil {
rand.Seed(int64(bs.ByteSlice(seed).Int64()))
}
}
}
func TestPutHasGetRemoveLinearHash(t *testing.T) {
fmt.Println("start test")
const RECORDS = 300
g := testfile(t, VPATH)
defer func() {
if e := g.Close(); e != nil {
panic(e)
}
if e := g.Remove(); e != nil {
panic(e)
}
}()
store, err := bucket.NewVarcharStore(g)
if err != nil {
panic(err)
}
f := testfile(t, PATH)
defer func() {
if e := f.Close(); e != nil {
panic(e)
}
if e := f.Remove(); e != nil {
panic(e)
}
}()
linhash, err := NewLinearHash(f, store)
if err != nil {
t.Fatal(err)
}
type record struct {
key bs.ByteSlice
value bs.ByteSlice
}
keyset := make(map[string]bool)
var records []*record
var values2 []bs.ByteSlice
for i := 0; i < RECORDS; i++ {
key := randslice(rand.Intn(16) + 1)
for {
if _, has := keyset[string(key)]; !has {
break
}
key = randslice(rand.Intn(16) + 1)
}
keyset[string(key)] = true
records = append(records, &record{key, randslice(rand.Intn(150) + 25)})
values2 = append(values2, randslice(rand.Intn(150)+25))
}
fmt.Println("real start test")
for i, record := range records {
err := linhash.Put(record.key, record.value)
if err != nil {
t.Fatal(err)
}
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
hash := hash(record.key)
bkt_idx := linhash.bucket(hash)
bkt, _ := linhash.get_bucket(bkt_idx)
bkt.PrintBucket()
bkt_idx2 := bkt_idx - (1 << (linhash.ctrl.i - 1))
if bkt_idx2 < linhash.ctrl.buckets {
bkt2, _ := linhash.get_bucket(bkt_idx2)
bkt2.PrintBucket()
}
fmt.Println(i, bs.ByteSlice64(hash), record.key, bkt_idx, bkt_idx2, linhash.ctrl.buckets, linhash.ctrl.i)
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(record.value) {
t.Fatal("Error getting record, value was not as expected")
}
}
if linhash.ctrl.records != RECORDS {
t.Fatalf("Expected record count == %d got %d", RECORDS,
linhash.ctrl.records)
}
rkeys, err := linhash.Keys()
if err != nil {
t.Fatal(err)
}
rkeyset := make(map[string]bool)
for _, bkey := range rkeys {
key := string(bkey)
if _, has := keyset[key]; !has {
t.Fatal("got non-existent key", bs.ByteSlice(key))
}
rkeyset[key] = true
}
for key, _ := range keyset {
if _, has := rkeyset[key]; !has {
t.Fatal("missed key", bs.ByteSlice(key))
}
}
for i, record := range records {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
hash := hash(record.key)
bkt_idx := linhash.bucket(hash)
bkt, _ := linhash.get_bucket(bkt_idx)
bkt.PrintBucket()
bkt_idx2 := bkt_idx - (1 << (linhash.ctrl.i - 1))
if bkt_idx2 < linhash.ctrl.buckets {
bkt2, _ := linhash.get_bucket(bkt_idx2)
bkt2.PrintBucket()
}
fmt.Println(i, bs.ByteSlice64(hash), record.key, bkt_idx, bkt_idx2, linhash.ctrl.buckets, linhash.ctrl.i)
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(record.value) {
t.Fatal("Error getting record, value was not as expected")
}
ran := randslice(rand.Intn(25) + 1)
if _, has := keyset[string(ran)]; !has {
value, err := linhash.DefaultGet(ran, bs.ByteSlice64(0))
if err != nil {
t.Fatal(err)
}
if !value.Eq(bs.ByteSlice64(0)) {
t.Fatal("Error getting default")
}
} else {
_, err := linhash.DefaultGet(ran, bs.ByteSlice64(0))
if err != nil {
t.Fatal(err)
}
}
}
for i, record := range records {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(record.value) {
t.Fatal("Error getting record, value was not as expected")
}
err = linhash.Put(record.key, values2[i])
if err != nil {
t.Fatal(err)
}
value, err = linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(values2[i]) {
t.Fatal("Error getting record, value was not as expected")
}
if linhash.Length() != RECORDS {
t.Fatalf("Expected record count == %d got %d", RECORDS,
linhash.ctrl.records)
}
}
length := linhash.Length()
for _, record := range records[length/2:] {
err := linhash.Remove(record.key)
if err != nil {
t.Fatal(err)
}
}
for _, record := range records[length/2:] {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("expected key to be gone")
}
}
for i, record := range records[:length/2] {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(values2[i]) {
t.Fatal("Error getting record, value was not as expected")
}
}
for _, record := range records[:length/2] {
err := linhash.Remove(record.key)
if err != nil {
t.Fatal(err)
}
}
for _, record := range records {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("expected key to be gone")
}
}
if linhash.Length() != 0 {
t.Fatalf("Expected record count == %d got %d", 0,
linhash.ctrl.records)
}
}
func TestPutHasGetRemove(t *testing.T) {
type record struct {
key String
value String
}
ranrec := func() *record {
return &record{
String(bs.ByteSlice(randstr(20)).String()),
String(bs.ByteSlice(randstr(20)).String()),
}
}
test := func(table HashTable) {
records := make([]*record, 400)
for i := range records {
r := ranrec()
records[i] = r
err := table.Put(r.key, String(""))
if err != nil {
t.Error(err)
}
err = table.Put(r.key, r.value)
if err != nil {
t.Error(err)
}
if table.Size() != (i + 1) {
t.Error("size was wrong", table.Size(), i+1)
}
}
for _, r := range records {
if has := table.Has(r.key); !has {
t.Error(table, "Missing key")
}
if has := table.Has(randstr(12)); has {
t.Error("Table has extra key")
}
if val, err := table.Get(r.key); err != nil {
t.Error(err)
} else if !(val.(String)).Equals(r.value) {
t.Error("wrong value")
}
}
for i, x := range records {
if val, err := table.Remove(x.key); err != nil {
t.Error(err)
} else if !(val.(String)).Equals(x.value) {
t.Error("wrong value")
}
for _, r := range records[i+1:] {
if has := table.Has(r.key); !has {
t.Error("Missing key")
}
if has := table.Has(randstr(12)); has {
t.Error("Table has extra key")
}
if val, err := table.Get(r.key); err != nil {
t.Error(err)
} else if !(val.(String)).Equals(r.value) {
t.Error("wrong value")
}
}
if table.Size() != (len(records) - (i + 1)) {
t.Error("size was wrong", table.Size(), (len(records) - (i + 1)))
}
}
}
test(NewHashTable(64))
test(NewLinearHash())
}
func TestPutHasGetRemoveBucket(t *testing.T) {
type record struct {
key String
value String
}
records := make([]*record, 400)
var tree *bst
var err error
var val interface{}
var updated bool
ranrec := func() *record {
return &record{randstr(20), randstr(20)}
}
for i := range records {
r := ranrec()
records[i] = r
tree, updated = tree.Put(r.key, String(""))
if updated {
t.Error("should have not been updated")
}
tree, updated = tree.Put(r.key, r.value)
if !updated {
t.Error("should have been updated")
}
if tree.Size() != (i + 1) {
t.Error("size was wrong", tree.Size(), i+1)
}
}
for _, r := range records {
if has := tree.Has(r.key); !has {
t.Error("Missing key")
}
if has := tree.Has(randstr(12)); has {
t.Error("Table has extra key")
}
if val, err := tree.Get(r.key); err != nil {
t.Error(err, bs.ByteSlice(val.(String)), bs.ByteSlice(r.value))
} else if !(val.(String)).Equals(r.value) {
t.Error("wrong value")
}
}
for i, x := range records {
if tree, val, err = tree.Remove(x.key); err != nil {
t.Error(err)
} else if !(val.(String)).Equals(x.value) {
t.Error("wrong value")
}
for _, r := range records[i+1:] {
if has := tree.Has(r.key); !has {
t.Error("Missing key")
}
if has := tree.Has(randstr(12)); has {
t.Error("Table has extra key")
}
if val, err := tree.Get(r.key); err != nil {
t.Error(err)
} else if !(val.(String)).Equals(r.value) {
t.Error("wrong value")
}
}
if tree.Size() != (len(records) - (i + 1)) {
t.Error("size was wrong", tree.Size(), (len(records) - (i + 1)))
}
}
}
func randstr(length int) types.String {
return types.String(bs.ByteSlice(randslice(length)).String()[2:])
}
func (self *VarcharList) Push(key int64, raw_bytes bs.ByteSlice) (err error) {
// fmt.Println()
element := new_element(raw_bytes)
bytes := element.Bytes()
item_key, hblk, dirty, blocks, err := self.alloc(key, int64(len(bytes)))
if err != nil {
return err
}
start_offset := int(self.data_offset(item_key))
end_offset := int(self.data_offset(self._find_end_algo(blocks, item_key, uint32(len(bytes)))))
// fmt.Println( "VarcharList.Push", key, len(blocks[0].data), start_offset, end_offset, len(bytes))
if len(blocks) == 1 {
// fmt.Println("VarcharList.Push", "only one block just copy")
copy(blocks[0].data[start_offset:end_offset], bytes)
} else {
// fmt.Println("VarcharList.Push", "several blocks")
var strings []string
start_bytes_offset := len(blocks[0].data) - start_offset
s := fmt.Sprint("start ", len(blocks), len(bytes), start_offset, start_bytes_offset)
strings = append(strings, s)
copy(blocks[0].data[start_offset:], bytes[0:start_bytes_offset])
offset := start_bytes_offset
for i, blk := range blocks[1 : len(blocks)-1] {
s := fmt.Sprint("middle ", i, len(blocks), len(bytes), offset, offset+len(blk.data))
strings = append(strings, s)
copy(blk.data, bytes[offset:offset+len(blk.data)])
offset += len(blk.data)
}
if offset > len(bytes) {
fmt.Println(len(blocks), len(bytes), offset)
panic(fmt.Errorf("offset out of bounds on bytes"))
}
if end_offset > len(blocks[len(blocks)-1].data) {
for _, s := range strings {
fmt.Println(s)
}
fmt.Println(len(blocks), len(bytes), len(blocks[len(blocks)-1].data), end_offset)
panic(fmt.Errorf("offset out of bounds on blocks[len(blocks)-1].data")) // this is the trigger!
}
copy(blocks[len(blocks)-1].data[:end_offset], bytes[offset:]) // BUG HERE
}
hblk.header.list_length += 1
err = dirty.Write()
if err != nil {
return err
}
start_blk := blocks[0]
offset := item_key - start_blk.key
ramt := start_blk.bytes[offset : offset+4].Int32()
ramt2 := start_blk.data[self.data_offset(item_key) : self.data_offset(item_key)+4].Int32()
// fmt.Println("VarcharList.Push", "cal_offset", offset, self.data_offset(item_key), self.data_offset(item_key) + 40)
if ramt != element.length() || ramt2 != ramt {
panic(fmt.Errorf("Written amount incorrect! %v != %v, %v \n %v", element.length(), ramt, bs.ByteSlice(element.Bytes()), blocks[0].data))
}
return nil
}