func (l *List) Swap(i, j uint64) (err error) { if i >= l.count { return errors.Errorf("index, i, out of range") } else if j >= l.count { return errors.Errorf("index, j, out of range") } return l.blk(i, func(iIdx *idxBlk) (err error) { var ia uint64 var I uint16 = uint16(i % itemsPerIdx) ia, err = iIdx.Get(I) if err != nil { return err } return l.blk(j, func(jIdx *idxBlk) (err error) { var ja uint64 var J uint16 = uint16(j % itemsPerIdx) ja, err = jIdx.Get(J) if err != nil { return err } err = iIdx.Set(I, ja) if err != nil { return err } return jIdx.Set(J, ia) }) }) }
func (n *internal) putKey(v *Varchar, key []byte, put func(i int) error) (err error) { if n.keyCount()+1 >= int(n.meta.keyCap) { return errors.Errorf("Block is full.") } var i int var has bool if n.meta.flags&consts.VARCHAR_KEYS == 0 { i, has, err = find(v, n, key) } else { err = v.Do(*slice.AsUint64(&key), func(key []byte) (err error) { i, has, err = find(v, n, key) return err }) } if err != nil { return err } if i < 0 { return errors.Errorf("find returned a negative int") } else if i >= int(n.meta.keyCap) { return errors.Errorf("find returned a int > than len(keys)") } else if has { return errors.Errorf(fmt.Sprintf("would have inserted a duplicate key, %v", key)) } if err := n.putKeyAt(key, i); err != nil { return err } return put(i) }
/* returns the key at the address and index or an error */ func (self *BpTree) keyAt(a uint64, i int) (key []byte, err error) { err = self.do( a, func(n *internal) error { if i >= int(n.meta.keyCount) { return errors.Errorf("out of range") } return n.doKeyAt(self.varchar, i, func(k []byte) error { key = make([]byte, len(k)) copy(key, k) return nil }) }, func(n *leaf) error { if i >= int(n.meta.keyCount) { return errors.Errorf("out of range") } return n.doKeyAt(self.varchar, i, func(k []byte) error { key = make([]byte, len(k)) copy(key, k) return nil }) }, ) if err != nil { return nil, err } return key, nil }
func (v *Varchar) doFree(a uint64, do func(*varFree) error) error { return v.do( a, func(*varCtrl) error { return errors.Errorf("unexpected ctrl blk") }, do, func(*varRunMeta) error { return errors.Errorf("unexpected run blk") }, ) }
func (v *Varchar) doCtrl(do func(*varCtrl) error) error { return v.do( v.a, do, func(*varFree) error { return errors.Errorf("unexpected free blk") }, func(*varRunMeta) error { return errors.Errorf("unexpected run blk") }, ) }
// Remove the underlying file. (must be already closed). func (self *BlockFile) Remove() error { if self.opened { return errors.Errorf("Expected file to be closed") } if self.Path() == "" { return errors.Errorf("This was an anonymous map") } return os.Remove(self.Path()) }
func (n *internal) updateK(v *Varchar, i int, key []byte) error { if i < 0 || i >= int(n.meta.keyCount) { return errors.Errorf("key is out of range") } if len(key) != int(n.meta.keySize) { return errors.Errorf("key was the wrong size") } idx, has, err := find(v, n, key) if err != nil { return err } if has && i != idx { log.Println(n.Debug(v)) n.doKeyAt(v, idx, func(x []byte) error { k := n.key(idx) if v != nil { y, e := v.UnsafeGet(*slice.AsUint64(&k)) log.Println(e) log.Println("key", x, y) } else { log.Println("key", x, "v was nil") } return nil }) n.doKeyAt(v, i, func(x []byte) error { log.Println("replacing", x) return nil }) return errors.Errorf("internal already had key %v, at %v, was going to put it at %v, replacing %v", key, idx, i, n.key(i)) } oldk := make([]byte, len(n.key(i))) copy(oldk, n.key(i)) flags := n.meta.flags if flags&consts.VARCHAR_KEYS != 0 { err := n.bigUpdateK(v, i, key) if err != nil { return err } } else { copy(n.key(i), key) } /* err = checkOrder(v, n) if err != nil { log.Println("replaced key", oldk) log.Println(n.Debug(v)) return err } */ return nil }
func (n *internal) delKP(v *Varchar, key []byte) error { i, has, err := find(v, n, key) if err != nil { return err } if !has { return errors.Errorf("key was not in the internal node") } else if i < 0 { return errors.Errorf("find returned a negative int") } else if i >= int(n.meta.keyCount) { return errors.Errorf("find returned a int > than len(keys)") } return n.delItemAt(v, i) }
func (l *List) Pop() (item []byte, err error) { if l.count == 0 { return nil, errors.Errorf("Cannot pop an empty list") } var a uint64 err = l.lastBlk(func(idx *idxBlk) (err error) { a, err = idx.Pop() return err }) if err != nil { return nil, err } err = l.varchar.Do(a, func(data []byte) error { item = make([]byte, len(data)) copy(item, data) return nil }) if err != nil { return nil, err } err = l.varchar.Free(a) if err != nil { return nil, err } err = l.doCtrl(l.a, func(ctrl *ctrlBlk) error { ctrl.count-- l.count = ctrl.count return nil }) if err != nil { return nil, err } return item, nil }
func (a *internal) balance(v *Varchar, b *internal) error { if b.meta.keyCount != 0 { return errors.Errorf("b was not empty") } m, err := a.balancePoint(v) if err != nil { return err } var lim int = int(a.meta.keyCount) - m for i := 0; i < lim; i++ { j := m + i copy(b.key(i), a.key(j)) fmap.MemClr(a.key(j)) *b.ptr(i) = *a.ptr(j) *a.ptr(j) = 0 } b.meta.keyCount = a.meta.keyCount - uint16(m) a.meta.keyCount = uint16(m) /* err = checkOrder(v, a) if err != nil { log.Println("balance point", m) log.Println(a) return err } err = checkOrder(v, b) if err != nil { log.Println("balance point", m) log.Println(b) return err } */ return nil }
func (l *List) Set(i uint64, item []byte) (err error) { if i >= l.count { return errors.Errorf("index out of range") } var old_a uint64 err = l.blk(i, func(idx *idxBlk) (err error) { old_a, err = idx.Get(uint16(i % itemsPerIdx)) return err }) if err != nil { return err } err = l.varchar.Free(old_a) if err != nil { return err } a, err := l.varchar.Alloc(len(item)) if err != nil { return err } err = l.varchar.Do(a, func(data []byte) error { copy(data, item) return nil }) if err != nil { return err } return l.blk(i, func(idx *idxBlk) (err error) { err = idx.Set(uint16(i%itemsPerIdx), a) if err != nil { return err } return }) }
func (v *Varchar) listRemove(node uint64, doNode nodeDoer) error { if node == 0 { return errors.Errorf("0 offset for node (the removed node)") } return doNode(node, func(n *listNode) (err error) { if n.prev != 0 { err = doNode(n.prev, func(pn *listNode) error { pn.next = n.next return nil }) if err != nil { return nil } } if n.next != 0 { err = doNode(n.next, func(nn *listNode) error { nn.prev = n.prev return nil }) if err != nil { return nil } } n.prev = 0 n.next = 0 return nil }) }
func (v *Varchar) listInsert(node, prev, next uint64, doNode nodeDoer) error { if node == 0 { return errors.Errorf("0 offset for node (the inserted node)") } return doNode(node, func(n *listNode) error { if prev == 0 && next == 0 { n.prev = 0 n.next = 0 return nil } else if next == 0 { return doNode(prev, func(pn *listNode) error { n.next = 0 n.prev = prev pn.next = node return nil }) } else if prev == 0 { return doNode(next, func(nn *listNode) error { n.next = next n.prev = 0 nn.prev = node return nil }) } return doNode(prev, func(pn *listNode) error { return doNode(next, func(nn *listNode) error { n.next = next n.prev = prev pn.next = node nn.prev = node return nil }) }) }) }
// Get the bytes at the offset and block count. You probably want to use // Do instead. You must call Release() on the bytes when done. func (self *BlockFile) Get(offset, blocks uint64) ([]byte, error) { if !self.opened { return nil, errors.Errorf("File is not open") } length := blocks * uint64(self.blksize) if (offset + length) > uint64(self.size) { return nil, errors.Errorf("Get outside of the file, (%d) %d + %d > %d", offset+length, offset, length, self.size) } self.outstanding += 1 slice := &slice.Slice{ Array: unsafe.Pointer(uintptr(self.mmap) + uintptr(offset)), Len: int(length), Cap: int(length), } return *slice.AsBytes(), nil }
func (v *Varchar) UnsafeGet(a uint64) (bytes []byte, err error) { rbytes, err := v.unsafeGet(a) if err != nil { return nil, err } m := asRunMeta(rbytes) fullLength := v.allocAmt(int(m.length)) blks := uint64(v.blksNeeded(fullLength)) offset, start, _ := v.startOffsetBlks(a) for offset+uint64(fullLength) >= blks*uint64(v.bf.BlockSize()) { blks++ } size, err := v.bf.Size() if err != nil { return nil, err } for start+blks*uint64(v.bf.BlockSize()) > uint64(size) { blks-- } allBytes, err := v.bf.Get(start, blks) if err != nil { return nil, err } err = v.bf.Release(allBytes) if err != nil { return nil, err } bytes = allBytes[offset:] flags := consts.AsFlag(bytes) if flags&consts.VARCHAR_RUN == 0 { return nil, errors.Errorf("bad address, was not a run block") } r := asRun(bytes) return r.bytes[:r.meta.length], nil }
// Interact with the contents of the varchar. The bytes passed into the // callback are UNSAFE. You could cause a segmentation fault if you // simply copy the *slice* out of the function. You need to copy the // data instead. // // The right way: // // var myBytes []byte // err = v.Do(a, func(bytes []byte) error { // myBytes = make([]byte, len(bytes)) // copy(myBytes, bytes) // return nil // }) // if err != nil { // log.Fatal(err) // } // // you can of course interact with the bytes in the callback in any way // you want as long as no pointers escape. You can even change the // values of the bytes (and these changes will be persisted). However, // you cannot change the length of the varchar. func (v *Varchar) Do(a uint64, do func([]byte) error) (err error) { return v.doRun(a, func(m *varRunMeta) error { fullLength := v.allocAmt(int(m.length)) blks := uint64(v.blksNeeded(fullLength)) offset, start, _ := v.startOffsetBlks(a) for offset+uint64(fullLength) >= blks*uint64(v.bf.BlockSize()) { blks++ } size, err := v.bf.Size() if err != nil { return err } for start+blks*uint64(v.bf.BlockSize()) > uint64(size) { blks-- } return v.bf.Do(start, blks, func(bytes []byte) error { bytes = bytes[offset:] flags := consts.AsFlag(bytes) if flags&consts.VARCHAR_RUN == 0 { return errors.Errorf("bad address, was not a run block") } r := asRun(bytes) return do(r.bytes[:r.meta.length]) }) }) }
// Open a varchar structure in the given blockfile with the given offset // as the control block. This function will confirm that the control // block is indeed a properly formated control block. func OpenVarchar(bf *fmap.BlockFile, a uint64) (v *Varchar, err error) { v = &Varchar{bf: bf, a: a, blkSize: bf.BlockSize()} var ptOff uint64 var szOff uint64 err = v.bf.Do(v.a, 1, func(bytes []byte) error { ctrl := asCtrl(bytes) if ctrl.flags&consts.VARCHAR_CTRL == 0 { return errors.Errorf("Expected a Varchar control block") } ptOff = ctrl.posTree szOff = ctrl.sizeTree return nil }) if err != nil { return nil, err } v.posTree, err = OpenAt(bf, ptOff) if err != nil { return nil, err } v.sizeTree, err = OpenAt(bf, szOff) if err != nil { return nil, err } return v, nil }
func (self *BpTree) delListNode(node uint64) (err error) { if node == 0 { return errors.Errorf("0 offset for n") } return self.doLeaf(node, func(n *leaf) (err error) { if n.meta.prev != 0 { err = self.doLeaf(n.meta.prev, func(pn *leaf) (err error) { pn.meta.next = n.meta.next return nil }) if err != nil { return err } } if n.meta.next != 0 { err = self.doLeaf(n.meta.next, func(nn *leaf) (err error) { nn.meta.prev = n.meta.prev return nil }) if err != nil { return err } } n.meta.prev = 0 n.meta.next = 0 return nil }) }
// Allocate 1 block and return its offset. func (self *BlockFile) Allocate() (offset uint64, err error) { if !self.opened { return 0, errors.Errorf("File is not open") } var resize bool = false err = self.ctrl(func(ctrl *ctrlblk) error { var err error if ctrl.meta.free_len > 0 { offset, err = self.pop_free() } else { resize = true } return err }) if err != nil { return 0, err } if resize { offset, err = self.allocOne() if err != nil { return 0, err } } return self.zero(offset, 1) }
func (a *leaf) balance(v *Varchar, b *leaf) error { if b.meta.keyCount != 0 { return errors.Errorf("b was not empty") } m, err := a.balancePoint(v) if err != nil { return err } if m == 0 { // we had a pure balance return nil } err = a.balanceAt(b, m) if err != nil { return err } /* err = checkOrder(v, a) if err != nil { log.Println("balance point", m) log.Println(a) return err } err = checkOrder(v, b) if err != nil { log.Println("balance point", m) log.Println(b) return err } */ return nil }
func (self *BpTree) pureRun(start uint64) ([]uint64, error) { /* this algorithm was pretty tricky to port do to the "interesting" * memory management scheme that I am employing (everything inside * of "do" methods). Basically, the problem was it was mutating * variables which I can not mutate because they are stuck inside of * the do methods. I had to find a way to hoist just the information * I needed. */ var key []byte err := self.doLeaf(start, func(n *leaf) error { if n.meta.keyCount < 0 { return errors.Errorf("block was empty") } return n.doKeyAt(self.varchar, 0, func(k []byte) error { key = make([]byte, len(k)) copy(key, k) return nil }) }) if err != nil { return nil, err } getNext := func(a uint64) (next uint64, err error) { err = self.doLeaf(a, func(n *leaf) error { next = n.meta.next return nil }) return next, err } isValid := func(a uint64) (valid bool, err error) { if a == 0 { return false, nil } err = self.doLeaf(a, func(n *leaf) error { return n.doKeyAt(self.varchar, 0, func(k []byte) error { valid = bytes.Equal(key, k) return nil }) }) return valid, err } cur := start valid, err := isValid(cur) if err != nil { return nil, err } run := make([]uint64, 0, 10) for valid { run = append(run, cur) cur, err = getNext(cur) if err != nil { return nil, err } valid, err = isValid(cur) if err != nil { return nil, err } } return run, nil }
func (self *BpTree) leafVerify(parent uint64, idx int, n, sibling uint64) (err error) { a := n return self.doLeaf(a, func(n *leaf) error { if n.keyCount() == 0 { if parent != 0 { log.Println("warn, keyCount == 0", a, parent, sibling) } return nil } if n.pure(self.varchar) { return self.pureVerify(parent, idx, a, sibling) } err := self.leafOrderVerify(parent, idx, a, sibling) if err != nil { log.Println("error in leafVerify") log.Printf("out of order") log.Println("leaf", a, n.Debug(self.varchar)) return err } if n.meta.next != sibling { log.Println("error in leafVerify") log.Println("n.meta.next != sibling", n.meta.next, sibling) self.doLeaf(n.meta.next, func(m *leaf) error { log.Println("a", a, n.Debug(self.varchar)) log.Println("a.meta.next", n.meta.next, m.Debug(self.varchar)) return self.doLeaf(sibling, func(o *leaf) error { log.Println("sibling", sibling, o.Debug(self.varchar)) return nil }) }) return errors.Errorf("n.meta.next (%v) != sibling (%v)", n.meta.next, sibling) } return nil }) }
func (b *idxBlk) Set(i uint16, a uint64) error { if i < 0 || i >= b.count { return errors.Errorf("Idx out of range for Set") } b.items[i] = a return nil }
func loadLeaf(backing []byte) (*leaf, error) { n := asLeaf(backing) if n.meta.flags&consts.LEAF == 0 { return nil, errors.Errorf("Was not a leaf node") } return n, nil }
func loadInternal(backing []byte) (*internal, error) { n := asInternal(backing) if n.meta.flags&consts.INTERNAL == 0 { return nil, errors.Errorf("Was not an internal node") } return n, nil }
func (self *BpTree) firstKey(a uint64, do func(key []byte) error) error { return self.do( a, func(n *internal) error { if int(n.meta.keyCount) == 0 { return errors.Errorf("Block was empty") } return do(n.key(0)) }, func(n *leaf) error { if int(n.meta.keyCount) == 0 { return errors.Errorf("Block was empty") } return do(n.key(0)) }, ) }
func (b *idxBlk) Append(a uint64) error { if b.count+1 > uint16(len(b.items)) { return errors.Errorf("Could not append to idxBlk, blk full") } b.items[b.count] = a b.count++ return nil }
func do_map(f *os.File) (unsafe.Pointer, error) { var mmap unsafe.Pointer = unsafe.Pointer(uintptr(0)) errno := C.create_mmap(&mmap, C.int(f.Fd())) if errno != 0 { return nil, errors.Errorf("Could not create map fd = %d, %d", f.Fd(), errno) } return mmap, nil }
func do_anon_map(length uint32) (unsafe.Pointer, error) { var mmap unsafe.Pointer = unsafe.Pointer(uintptr(0)) errno := C.create_anon_mmap(&mmap, C.size_t(length)) if errno != 0 { return nil, errors.Errorf("Could not create anon map. length = %d, %d", length, errno) } return mmap, nil }
func (self *BpTree) doLeaf(a uint64, do func(*leaf) error) error { return self.do( a, func(n *internal) error { return errors.Errorf("Unexpected internal node") }, do, ) }