func (self *ctrlblk) Bytes() []byte {
bytes := make([]byte, CONTROLSIZE)
copy(bytes[0:8], bs.ByteSlice64(uint64(self.end)))
copy(bytes[8:16], bs.ByteSlice64(uint64(self.free_head)))
copy(bytes[16:20], bs.ByteSlice32(uint32(self.free_len)))
return bytes
}
func (self *ctrlblk) Bytes() []byte {
bytes := make([]byte, CONTROLSIZE)
copy(bytes[0:4], bs.ByteSlice32(self.buckets))
copy(bytes[4:12], bs.ByteSlice64(self.records))
copy(bytes[12:20], bs.ByteSlice64(uint64(self.table)))
bytes[20] = self.i
return bytes
}
func (self *list_header) Bytes() []byte {
bytes := make(bs.ByteSlice, LIST_HEADER_LEN)
copy(bytes[0:8], bs.ByteSlice64(uint64(self.next)))
copy(bytes[8:16], bs.ByteSlice64(uint64(self.head)))
copy(bytes[16:24], bs.ByteSlice64(uint64(self.tail)))
copy(bytes[24:32], bs.ByteSlice64(uint64(self.insert_point)))
copy(bytes[32:36], bs.ByteSlice32(self.block_count))
copy(bytes[36:40], bs.ByteSlice32(self.list_length))
return bytes
}
func NewLinearHash(file file.BlockDevice, kv bucket.KVStore) (self *LinearHash, err error) {
const NUMBUCKETS = 16
const I = 5
table, err := bucket.NewBlockTable(file, 4, 8)
if err != nil {
return nil, err
}
for n := uint32(0); n < NUMBUCKETS; n++ {
bkt, err := bucket.NewHashBucket(file, HASHSIZE, kv)
if err != nil {
return nil, err
}
err = table.Put(bs.ByteSlice32(n), bs.ByteSlice64(uint64(bkt.Key())))
if err != nil {
return nil, err
}
}
self = &LinearHash{
file: file,
kv: kv,
table: table,
ctrl: ctrlblk{
buckets: NUMBUCKETS,
records: 0,
table: table.Key(),
i: I,
},
}
return self, self.write_ctrlblk()
}
func (self *VarcharStore) Put(key, value bs.ByteSlice) (bytes bs.ByteSlice, err error) {
vc := self.varchar
kv := new_varchar_kv(key, value).Bytes()
vkey, err := vc.Write(kv)
if err != nil {
return nil, err
}
return bs.ByteSlice64(uint64(vkey)), nil
}
func (self *LinearHash) Get(key bs.ByteSlice) (value bs.ByteSlice, err error) {
hash := hash(key)
bkt_idx := self.bucket(hash)
bkt, err := self.get_bucket(bkt_idx)
if err != nil {
return nil, err
}
return bkt.Get(bs.ByteSlice64(hash), key)
}
func (self *Varchar) set_length(key int64, blk *block, length uint64) (err error) {
block_size := datasize(self.file)
offset := self.block_offset(key)
if offset > block_size {
return fmt.Errorf("Would write length off the end of the block")
}
copy(blk.data[offset:offset+LENSIZE], bs.ByteSlice64(length))
return nil
}
func (self *header) Bytes() (bytes []byte) {
bytes = make([]byte, HEADER_SIZE)
bytes[0] = self.flag
bytes[1] = self.keysize
bytes[2] = self.valsize
copy(bytes[4:8], bs.ByteSlice32(self.records))
copy(bytes[8:16], bs.ByteSlice64(uint64(self.next)))
copy(bytes[16:18], bs.ByteSlice16(self.blocks))
return bytes
}
func (self *LinearHash) Has(key bs.ByteSlice) (has bool, err error) {
hash := hash(key)
bkt_idx := self.bucket(hash)
bkt, err := self.get_bucket(bkt_idx)
if err != nil {
return false, err
}
has = bkt.Has(bs.ByteSlice64(hash), key)
// fmt.Println("LinearHash.Has", bs.ByteSlice64(hash), key, bkt_idx, has)
return has, nil
}
func (self *LinearHash) split() (err error) {
bkt_idx := self.ctrl.buckets % (1 << (self.ctrl.i - 1))
bkt, err := self.get_bucket(bkt_idx)
if err != nil {
return err
}
keys := bkt.Keys()
self.ctrl.buckets += 1
if self.ctrl.buckets > (1 << self.ctrl.i) {
self.ctrl.i += 1
}
newbkt, err := bkt.Split(func(key bs.ByteSlice) bool {
this_idx := self.bucket(key.Int64())
return this_idx == bkt_idx
})
err = self.table.Put(bs.ByteSlice32(self.ctrl.buckets-1), bs.ByteSlice64(uint64(newbkt.Key())))
if err != nil {
return err
}
err = self.write_ctrlblk()
if err != nil {
return err
}
for _, key := range keys {
if has, err := self.Has(key); err != nil {
return err
} else if !has {
hash := bs.ByteSlice64(hash(key))
in_first := bkt.Has(hash, key)
in_second := newbkt.Has(hash, key)
fmt.Println()
fmt.Println("i", self.ctrl.i, "records", self.ctrl.records, "buckets", self.ctrl.buckets)
fmt.Println("key", key, "hash", hash)
fmt.Println("in_first", in_first, bkt_idx)
fmt.Println("in_second", in_second, self.ctrl.buckets-1)
return fmt.Errorf("Key went missing during split")
}
}
return nil
}
func (self *LinearHash) DefaultGet(key bs.ByteSlice, default_value bs.ByteSlice) (value bs.ByteSlice, err error) {
hash := hash(key)
hash_bytes := bs.ByteSlice64(hash)
bkt_idx := self.bucket(hash)
bkt, err := self.get_bucket(bkt_idx)
if err != nil {
return nil, err
}
if bkt.Has(hash_bytes, key) {
return bkt.Get(hash_bytes, key)
}
return default_value, nil
}
func (self *LinearHash) Remove(key bs.ByteSlice) (err error) {
hash := hash(key)
bkt_idx := self.bucket(hash)
bkt, err := self.get_bucket(bkt_idx)
if err != nil {
return err
}
err = bkt.Remove(bs.ByteSlice64(hash), key)
if err != nil {
return err
}
self.ctrl.records -= 1
return self.write_ctrlblk()
}
func (self *VarcharStore) Update(bytes, key, value bs.ByteSlice) (rbytes bs.ByteSlice, err error) {
vc := self.varchar
vkey := int64(bytes.Int64())
kv := new_varchar_kv(key, value).Bytes()
oldkv, err := vc.Read(vkey)
if err != nil {
return nil, err
}
if len(oldkv) == len(kv) {
err = vc.Update(vkey, kv)
if err != nil {
return nil, err
}
return bs.ByteSlice64(uint64(vkey)), nil
} else {
err := vc.Remove(vkey)
if err != nil {
return nil, err
}
return self.Put(key, value)
}
}
func (self *LinearHash) Put(key bs.ByteSlice, value bs.ByteSlice) (err error) {
hash := hash(key)
bkt_idx := self.bucket(hash)
bkt, err := self.get_bucket(bkt_idx)
if err != nil {
fmt.Println("Couldn't get bucket idx", bkt_idx)
return err
}
updated, err := bkt.Put(bs.ByteSlice64(hash), key, value)
if err != nil {
return err
}
if !updated {
self.ctrl.records += 1
if self.split_needed() {
// fmt.Println("did split")
return self.split()
}
// fmt.Println("no split")
return self.write_ctrlblk()
}
return nil
}
func TestPutHasGetRemoveLinearHash(t *testing.T) {
fmt.Println("start test")
const RECORDS = 300
g := testfile(t, VPATH)
defer func() {
if e := g.Close(); e != nil {
panic(e)
}
if e := g.Remove(); e != nil {
panic(e)
}
}()
store, err := bucket.NewVarcharStore(g)
if err != nil {
panic(err)
}
f := testfile(t, PATH)
defer func() {
if e := f.Close(); e != nil {
panic(e)
}
if e := f.Remove(); e != nil {
panic(e)
}
}()
linhash, err := NewLinearHash(f, store)
if err != nil {
t.Fatal(err)
}
type record struct {
key bs.ByteSlice
value bs.ByteSlice
}
keyset := make(map[string]bool)
var records []*record
var values2 []bs.ByteSlice
for i := 0; i < RECORDS; i++ {
key := randslice(rand.Intn(16) + 1)
for {
if _, has := keyset[string(key)]; !has {
break
}
key = randslice(rand.Intn(16) + 1)
}
keyset[string(key)] = true
records = append(records, &record{key, randslice(rand.Intn(150) + 25)})
values2 = append(values2, randslice(rand.Intn(150)+25))
}
fmt.Println("real start test")
for i, record := range records {
err := linhash.Put(record.key, record.value)
if err != nil {
t.Fatal(err)
}
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
hash := hash(record.key)
bkt_idx := linhash.bucket(hash)
bkt, _ := linhash.get_bucket(bkt_idx)
bkt.PrintBucket()
bkt_idx2 := bkt_idx - (1 << (linhash.ctrl.i - 1))
if bkt_idx2 < linhash.ctrl.buckets {
bkt2, _ := linhash.get_bucket(bkt_idx2)
bkt2.PrintBucket()
}
fmt.Println(i, bs.ByteSlice64(hash), record.key, bkt_idx, bkt_idx2, linhash.ctrl.buckets, linhash.ctrl.i)
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(record.value) {
t.Fatal("Error getting record, value was not as expected")
}
}
if linhash.ctrl.records != RECORDS {
t.Fatalf("Expected record count == %d got %d", RECORDS,
linhash.ctrl.records)
}
rkeys, err := linhash.Keys()
if err != nil {
t.Fatal(err)
}
rkeyset := make(map[string]bool)
for _, bkey := range rkeys {
key := string(bkey)
if _, has := keyset[key]; !has {
t.Fatal("got non-existent key", bs.ByteSlice(key))
}
rkeyset[key] = true
}
for key, _ := range keyset {
if _, has := rkeyset[key]; !has {
t.Fatal("missed key", bs.ByteSlice(key))
}
}
for i, record := range records {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
hash := hash(record.key)
bkt_idx := linhash.bucket(hash)
bkt, _ := linhash.get_bucket(bkt_idx)
bkt.PrintBucket()
bkt_idx2 := bkt_idx - (1 << (linhash.ctrl.i - 1))
if bkt_idx2 < linhash.ctrl.buckets {
bkt2, _ := linhash.get_bucket(bkt_idx2)
bkt2.PrintBucket()
}
fmt.Println(i, bs.ByteSlice64(hash), record.key, bkt_idx, bkt_idx2, linhash.ctrl.buckets, linhash.ctrl.i)
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(record.value) {
t.Fatal("Error getting record, value was not as expected")
}
ran := randslice(rand.Intn(25) + 1)
if _, has := keyset[string(ran)]; !has {
value, err := linhash.DefaultGet(ran, bs.ByteSlice64(0))
if err != nil {
t.Fatal(err)
}
if !value.Eq(bs.ByteSlice64(0)) {
t.Fatal("Error getting default")
}
} else {
_, err := linhash.DefaultGet(ran, bs.ByteSlice64(0))
if err != nil {
t.Fatal(err)
}
}
}
for i, record := range records {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(record.value) {
t.Fatal("Error getting record, value was not as expected")
}
err = linhash.Put(record.key, values2[i])
if err != nil {
t.Fatal(err)
}
value, err = linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(values2[i]) {
t.Fatal("Error getting record, value was not as expected")
}
if linhash.Length() != RECORDS {
t.Fatalf("Expected record count == %d got %d", RECORDS,
linhash.ctrl.records)
}
}
length := linhash.Length()
for _, record := range records[length/2:] {
err := linhash.Remove(record.key)
if err != nil {
t.Fatal(err)
}
}
for _, record := range records[length/2:] {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("expected key to be gone")
}
}
for i, record := range records[:length/2] {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("Expected key")
}
value, err := linhash.Get(record.key)
if err != nil {
t.Fatal(err)
}
if !value.Eq(values2[i]) {
t.Fatal("Error getting record, value was not as expected")
}
}
for _, record := range records[:length/2] {
err := linhash.Remove(record.key)
if err != nil {
t.Fatal(err)
}
}
for _, record := range records {
has, err := linhash.Has(record.key)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("expected key to be gone")
}
}
if linhash.Length() != 0 {
t.Fatalf("Expected record count == %d got %d", 0,
linhash.ctrl.records)
}
}
func (self *metadata) Bytes() []byte {
bytes := make([]byte, METADATASIZE)
copy(bytes[0:8], bs.ByteSlice64(uint64(self.next)))
return bytes
}
func (self *free_varchar) Bytes() []byte {
bytes := make([]byte, FREE_VARCHAR_SIZE)
copy(bytes[0:8], bs.ByteSlice64(self.length))
copy(bytes[8:16], bs.ByteSlice64(uint64(self.next)))
return bytes
}