func (blob *Blob) saveChunk(key cas.Key, level uint8) (cas.Key, error) {
if !key.IsPrivate() {
// already saved
return key, nil
}
chunk, err := blob.stash.Get(key, blob.m.Type, level)
if err != nil {
return key, err
}
if level > 0 {
for off := uint32(0); off+cas.KeySize <= uint32(len(chunk.Buf)); off += cas.KeySize {
cur := cas.NewKeyPrivate(chunk.Buf[off : off+cas.KeySize])
if cur.IsReserved() {
return key, fmt.Errorf("invalid stored key: key @%d in %v is %v", off, key, chunk.Buf[off:off+cas.KeySize])
}
// recurses at most `level` deep
saved, err := blob.saveChunk(cur, level-1)
if err != nil {
return key, err
}
copy(chunk.Buf[off:off+cas.KeySize], saved.Bytes())
}
}
chunk.Buf = trim(chunk.Buf)
return blob.stash.Save(key)
}
func TestKeyNewPrivateNum(t *testing.T) {
k := cas.NewKeyPrivateNum(31337)
buf := k.Bytes()
k2 := cas.NewKey(buf)
if g, e := k2, cas.Invalid; g != e {
t.Errorf("expected NewKey to give Invalid: %v", g)
}
k3 := cas.NewKeyPrivate(buf)
if g, e := k3, k; g != e {
t.Errorf("expected NewKeyPrivate to give original key: %v", g)
}
priv, ok := k3.Private()
if !ok {
t.Fatalf("expected Private to work: %v %v", priv, ok)
}
if g, e := priv, uint64(31337); g != e {
t.Errorf("expected Private to match original: %v", g)
}
if g, e := k.IsSpecial(), true; g != e {
t.Errorf("not Special: %v != %v", g, e)
}
if g, e := k.IsPrivate(), true; g != e {
t.Errorf("not Private: %v != %v", g, e)
}
if g, e := k.IsReserved(), false; g != e {
t.Errorf("bad Reserved: %v != %v", g, e)
}
}
// Decreases depth, always selecting only the leftmost tree,
// and dropping all Private chunks in the rest.
func (blob *Blob) shrink(level uint8) error {
for blob.depth > level {
chunk, err := blob.stash.Get(blob.m.Root, blob.m.Type, blob.depth)
if err != nil {
return err
}
if blob.m.Root.IsPrivate() {
// blob.depth must be >0 if we're here, so it's always a
// pointer chunk; iterate all non-first keys and drop
// Private chunks
err = blob.discardAfter(chunk, 1, blob.depth)
if err != nil {
return err
}
}
// now all non-left top-level private nodes have been dropped
keybuf := safeSlice(chunk.Buf, 0, cas.KeySize)
key := cas.NewKeyPrivate(keybuf)
blob.m.Root = key
blob.depth -= 1
}
return nil
}
// lookup fetches the data chunk for given global byte offset.
//
// The returned Chunk remains zero trimmed.
//
// It may be a Private or a normal chunk. For writable Chunks, call
// lookupForWrite instead.
func (blob *Blob) lookup(off uint64) (*chunks.Chunk, error) {
gidx := uint32(off / uint64(blob.m.ChunkSize))
lidxs := localChunkIndexes(blob.m.Fanout, gidx)
level := blob.depth
// walk down from the root
var ptrKey = blob.m.Root
for ; level > 0; level-- {
// follow pointer chunks
var idx uint32
if int(level)-1 < len(lidxs) {
idx = lidxs[level-1]
}
chunk, err := blob.stash.Get(ptrKey, blob.m.Type, level)
if err != nil {
return nil, err
}
keyoff := int64(idx) * cas.KeySize
// zero trimming may have cut the key off, even in the middle
// TODO ugly int conversion
keybuf := safeSlice(chunk.Buf, int(keyoff), int(keyoff+cas.KeySize))
ptrKey = cas.NewKeyPrivate(keybuf)
}
chunk, err := blob.stash.Get(ptrKey, blob.m.Type, 0)
return chunk, err
}
func TestKeyInvalidPrivate(t *testing.T) {
buf := make([]byte, cas.KeySize)
buf[len(buf)-1] = 0x42
k := cas.NewKeyPrivate(buf)
if g, e := k, cas.Invalid; g != e {
t.Errorf("not Invalid: %q != %q", g, e)
}
}
// chunk must be a Private chunk
func (blob *Blob) discardAfter(chunk *chunks.Chunk, lidx uint32, level uint8) error {
if level == 0 {
return nil
}
for ; lidx < blob.m.Fanout; lidx++ {
keyoff := lidx * cas.KeySize
keybuf := chunk.Buf[keyoff : keyoff+cas.KeySize]
key := cas.NewKeyPrivate(keybuf)
if key.IsPrivate() {
// there can't be any Private chunks if they key wasn't Private
chunk, err := blob.stash.Get(key, blob.m.Type, level-1)
if err != nil {
return err
}
err = blob.discardAfter(chunk, 0, level-1)
if err != nil {
return err
}
blob.stash.Drop(key)
}
copy(chunk.Buf[keyoff:keyoff+cas.KeySize], cas.Empty.Bytes())
}
return nil
}
// Truncate adjusts the size of the blob. If the new size is less than
// the old size, data past that point is lost. If the new size is
// greater than the old size, the new part is full of zeroes.
func (blob *Blob) Truncate(size uint64) error {
switch {
case size == 0:
// special case shrink to nothing
blob.m.Root = cas.Empty
blob.m.Size = 0
blob.stash.Clear()
case size < blob.m.Size:
// shrink
// i really am starting to hate the idea of file offsets being
// int64's, but can't fight all the windmills at once.
if size > math.MaxInt64 {
return errors.New("cannot discard past 63-bit file size")
}
// we know size>0 from above
off := size - 1
gidx := uint32(off / uint64(blob.m.ChunkSize))
lidxs := localChunkIndexes(blob.m.Fanout, gidx)
err := blob.shrink(uint8(len(lidxs)))
if err != nil {
return err
}
// we don't need to always cow here (if everything is
// perfectly aligned / already zero), but it's a rare enough
// case that let's not care for now
//
// TODO this makes a tight loop on Open and Save wasteful
{
// TODO clone all the way down to be able to trim leaf chunk,
// abusing lookupForWrite for now
// we know size > 0 from above
_, err := blob.lookupForWrite(size - 1)
if err != nil {
return err
}
}
// now zero-fill on the right; guaranteed cow by the above kludge
key := blob.m.Root
if debugTruncate {
if !key.IsPrivate() {
panic(fmt.Errorf("Truncate root is not private: %v", key))
}
}
for level := blob.depth; level > 0; level-- {
chunk, err := blob.stash.Get(key, blob.m.Type, level)
if err != nil {
return err
}
err = blob.discardAfter(chunk, lidxs[level-1]+1, level)
if err != nil {
return err
}
keyoff := int64(lidxs[level-1]) * cas.KeySize
keybuf := chunk.Buf[keyoff : keyoff+cas.KeySize]
key = cas.NewKeyPrivate(keybuf)
if debugTruncate {
if !key.IsPrivate() {
panic(fmt.Errorf("Truncate key at level %d not private: %v", level, key))
}
}
}
// and finally the leaf chunk
chunk, err := blob.stash.Get(key, blob.m.Type, 0)
if err != nil {
return err
}
{
// TODO is there anything to clear here; beware modulo wraparound
// size is also the offset of the next byte
loff := uint32(size % uint64(blob.m.ChunkSize))
zeroSlice(chunk.Buf[loff:])
}
// TODO what's the right time to adjust size, wrt errors
blob.m.Size = size
// TODO unit tests that checks we don't leak chunks?
case size > blob.m.Size:
// grow
off := size - 1
gidx := uint32(off / uint64(blob.m.ChunkSize))
lidxs := localChunkIndexes(blob.m.Fanout, gidx)
err := blob.grow(uint8(len(lidxs)))
if err != nil {
return err
}
blob.m.Size = size
}
return nil
}
// lookupForWrite fetches the data chunk for the given offset and
// ensures it is Private and reinflated, and thus writable.
func (blob *Blob) lookupForWrite(off uint64) (*chunks.Chunk, error) {
gidx := uint32(off / uint64(blob.m.ChunkSize))
lidxs := localChunkIndexes(blob.m.Fanout, gidx)
err := blob.grow(uint8(len(lidxs)))
if err != nil {
return nil, err
}
level := blob.depth
var parentChunk *chunks.Chunk
{
// clone root if necessary
var k cas.Key
var err error
size := blob.chunkSizeForLevel(level)
k, parentChunk, err = blob.stash.Clone(blob.m.Root, blob.m.Type, level, size)
if err != nil {
return nil, err
}
blob.m.Root = k
}
// walk down from the root
var ptrKey = blob.m.Root
for ; level > 0; level-- {
// follow pointer chunks
var idx uint32
if int(level)-1 < len(lidxs) {
idx = lidxs[level-1]
}
keyoff := int64(idx) * cas.KeySize
{
k := cas.NewKeyPrivate(parentChunk.Buf[keyoff : keyoff+cas.KeySize])
if k.IsReserved() {
return nil, fmt.Errorf("invalid stored key: key @%d in %v is %v", keyoff, ptrKey, parentChunk.Buf[keyoff:keyoff+cas.KeySize])
}
ptrKey = k
}
// clone it (nop if already cloned)
size := blob.chunkSizeForLevel(level - 1)
ptrKey, child, err := blob.stash.Clone(ptrKey, blob.m.Type, level-1, size)
if err != nil {
return nil, err
}
if debugLookup {
if uint64(len(child.Buf)) != uint64(size) {
panic(fmt.Errorf("lookupForWrite clone for level %d made weird size %d != %d, key %v", level-1, len(child.Buf), size, ptrKey))
}
}
// update the key in parent
n := copy(parentChunk.Buf[keyoff:keyoff+cas.KeySize], ptrKey.Bytes())
if debugLookup {
if n != cas.KeySize {
panic(fmt.Errorf("lookupForWrite copied only %d of the key", n))
}
}
parentChunk = child
}
if debugLookup {
if parentChunk.Level != 0 {
panic(fmt.Errorf("lookupForWrite got a non-leaf: %v", parentChunk.Level))
}
if uint64(len(parentChunk.Buf)) != uint64(blob.m.ChunkSize) {
panic(fmt.Errorf("lookupForWrite got short leaf: %v", len(parentChunk.Buf)))
}
}
return parentChunk, nil
}