// Returns RLEs for a given label where the key of the returned map is the block index
// in string format.
func (d *Data) GetLabelRLEs(v dvid.VersionID, label uint64) (dvid.BlockRLEs, error) {
store, err := storage.SmallDataStore()
if err != nil {
return nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
}
// Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys.
begIndex := NewTKey(label, dvid.MinIndexZYX.ToIZYXString())
endIndex := NewTKey(label, dvid.MaxIndexZYX.ToIZYXString())
// Process all the b+s keys and their values, which contain RLE runs for that label.
labelRLEs := dvid.BlockRLEs{}
var f storage.ChunkFunc = func(chunk *storage.Chunk) error {
// Get the block index where the fromLabel is present
_, blockStr, err := DecodeTKey(chunk.K)
if err != nil {
return fmt.Errorf("Can't recover block index with chunk key %v: %v\n", chunk.K, err)
}
var blockRLEs dvid.RLEs
if err := blockRLEs.UnmarshalBinary(chunk.V); err != nil {
return fmt.Errorf("Unable to unmarshal RLE for label in block %v", chunk.K)
}
labelRLEs[blockStr] = blockRLEs
return nil
}
ctx := datastore.NewVersionedCtx(d, v)
err = store.ProcessRange(ctx, begIndex, endIndex, &storage.ChunkOp{}, f)
if err != nil {
return nil, err
}
fmt.Printf("Found %d blocks with label %d\n", len(labelRLEs), label)
return labelRLEs, nil
}
// Alter serialized RLEs by the bounds.
func boundRLEs(b []byte, bounds *dvid.Bounds) ([]byte, error) {
var oldRLEs dvid.RLEs
err := oldRLEs.UnmarshalBinary(b)
if err != nil {
return nil, err
}
newRLEs := oldRLEs.FitToBounds(bounds)
return newRLEs.MarshalBinary()
}
func getBytesRLE(t *testing.T, rles dvid.RLEs) *bytes.Buffer {
n := len(rles)
buf := new(bytes.Buffer)
buf.WriteByte(dvid.EncodingBinary)
binary.Write(buf, binary.LittleEndian, uint8(3)) // # of dimensions
binary.Write(buf, binary.LittleEndian, byte(0)) // dimension of run (X = 0)
buf.WriteByte(byte(0)) // reserved for later
binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels
binary.Write(buf, binary.LittleEndian, uint32(n)) // Placeholder for # spans
rleBytes, err := rles.MarshalBinary()
if err != nil {
t.Errorf("Unable to serialize RLEs: %v\n", err)
}
buf.Write(rleBytes)
return buf
}
// SplitCoarseLabels splits a portion of a label's voxels into a given split label or, if the given split
// label is 0, a new label, which is returned. The input is a binary sparse volume defined by block
// coordinates and should be the smaller portion of a labeled region-to-be-split.
//
// EVENTS
//
// labels.SplitStartEvent occurs at very start of split and transmits labels.DeltaSplitStart struct.
//
// labels.SplitBlockEvent occurs for every block of a split label and transmits labels.DeltaSplit struct.
//
// labels.SplitEndEvent occurs at end of split and transmits labels.DeltaSplitEnd struct.
//
func (d *Data) SplitCoarseLabels(v dvid.VersionID, fromLabel, splitLabel uint64, r io.ReadCloser) (toLabel uint64, err error) {
store, err := d.GetOrderedKeyValueDB()
if err != nil {
err = fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
return
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
err = fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store)
return
}
// Create a new label id for this version that will persist to store
if splitLabel != 0 {
toLabel = splitLabel
dvid.Debugf("Splitting coarse subset of label %d into given label %d ...\n", fromLabel, splitLabel)
} else {
toLabel, err = d.NewLabel(v)
if err != nil {
return
}
dvid.Debugf("Splitting coarse subset of label %d into new label %d ...\n", fromLabel, toLabel)
}
evt := datastore.SyncEvent{d.DataUUID(), labels.SplitStartEvent}
splitOpStart := labels.DeltaSplitStart{fromLabel, toLabel}
splitOpEnd := labels.DeltaSplitEnd{fromLabel, toLabel}
// Make sure we can split given current merges in progress
if err := labels.SplitStart(d.getMergeIV(v), splitOpStart); err != nil {
return toLabel, err
}
defer labels.SplitStop(d.getMergeIV(v), splitOpEnd)
// Signal that we are starting a split.
msg := datastore.SyncMessage{labels.SplitStartEvent, v, splitOpStart}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return 0, err
}
// Read the sparse volume from reader.
var splits dvid.RLEs
splits, err = dvid.ReadRLEs(r)
if err != nil {
return
}
numBlocks, _ := splits.Stats()
// Order the split blocks
splitblks := make(dvid.IZYXSlice, numBlocks)
n := 0
for _, rle := range splits {
p := rle.StartPt()
run := rle.Length()
for i := int32(0); i < run; i++ {
izyx := dvid.IndexZYX{p[0] + i, p[1], p[2]}
splitblks[n] = izyx.ToIZYXString()
n++
}
}
sort.Sort(splitblks)
// Iterate through the split blocks, read the original block and change labels.
// TODO: Modifications should be transactional since it's GET-PUT, therefore use
// hash on block coord to direct it to block-specific goroutine; we serialize
// requests to handle concurrency.
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
var toLabelSize uint64
for _, splitblk := range splitblks {
// Get original block
tk := NewTKey(fromLabel, splitblk)
val, err := store.Get(ctx, tk)
if err != nil {
return toLabel, err
}
if val == nil {
return toLabel, fmt.Errorf("Split block %s is not part of original label %d", splitblk, fromLabel)
}
var rles dvid.RLEs
if err := rles.UnmarshalBinary(val); err != nil {
return toLabel, fmt.Errorf("Unable to unmarshal RLE for original labels in block %s", splitblk)
}
numVoxels, _ := rles.Stats()
toLabelSize += numVoxels
// Delete the old block and save the sparse volume but under a new label.
batch.Delete(tk)
tk2 := NewTKey(toLabel, splitblk)
batch.Put(tk2, val)
}
if err := batch.Commit(); err != nil {
dvid.Errorf("Batch PUT during split of %q label %d: %v\n", d.DataName(), fromLabel, err)
}
// Publish split event
evt = datastore.SyncEvent{d.DataUUID(), labels.SplitLabelEvent}
msg = datastore.SyncMessage{labels.SplitLabelEvent, v, labels.DeltaSplit{fromLabel, toLabel, nil, splitblks}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return 0, err
}
// Publish change in label sizes.
delta := labels.DeltaNewSize{
Label: toLabel,
Size: toLabelSize,
}
evt = datastore.SyncEvent{d.DataUUID(), labels.ChangeSizeEvent}
msg = datastore.SyncMessage{labels.ChangeSizeEvent, v, delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return 0, err
}
delta2 := labels.DeltaModSize{
Label: fromLabel,
SizeChange: int64(-toLabelSize),
}
evt = datastore.SyncEvent{d.DataUUID(), labels.ChangeSizeEvent}
msg = datastore.SyncMessage{labels.ChangeSizeEvent, v, delta2}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return 0, err
}
// Publish split end
evt = datastore.SyncEvent{d.DataUUID(), labels.SplitEndEvent}
msg = datastore.SyncMessage{labels.SplitEndEvent, v, splitOpEnd}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return 0, err
}
dvid.Infof("Split %d voxels from label %d to label %d\n", toLabelSize, fromLabel, toLabel)
return toLabel, nil
}
// SplitLabels splits a portion of a label's voxels into a given split label or, if the given split
// label is 0, a new label, which is returned. The input is a binary sparse volume and should
// preferably be the smaller portion of a labeled region. In other words, the caller should chose
// to submit for relabeling the smaller portion of any split. It is assumed that the given split
// voxels are within the fromLabel set of voxels and will generate unspecified behavior if this is
// not the case.
//
// EVENTS
//
// labels.SplitStartEvent occurs at very start of split and transmits labels.DeltaSplitStart struct.
//
// labels.SplitBlockEvent occurs for every block of a split label and transmits labels.DeltaSplit struct.
//
// labels.SplitEndEvent occurs at end of split and transmits labels.DeltaSplitEnd struct.
//
func (d *Data) SplitLabels(v dvid.VersionID, fromLabel, splitLabel uint64, r io.ReadCloser) (toLabel uint64, err error) {
store, err := d.GetOrderedKeyValueDB()
if err != nil {
err = fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
return
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
err = fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store)
return
}
// Create a new label id for this version that will persist to store
if splitLabel != 0 {
toLabel = splitLabel
dvid.Debugf("Splitting subset of label %d into given label %d ...\n", fromLabel, splitLabel)
} else {
toLabel, err = d.NewLabel(v)
if err != nil {
return
}
dvid.Debugf("Splitting subset of label %d into new label %d ...\n", fromLabel, toLabel)
}
evt := datastore.SyncEvent{d.DataUUID(), labels.SplitStartEvent}
splitOpStart := labels.DeltaSplitStart{fromLabel, toLabel}
splitOpEnd := labels.DeltaSplitEnd{fromLabel, toLabel}
// Make sure we can split given current merges in progress
if err := labels.SplitStart(d.getMergeIV(v), splitOpStart); err != nil {
return toLabel, err
}
defer labels.SplitStop(d.getMergeIV(v), splitOpEnd)
// Signal that we are starting a split.
msg := datastore.SyncMessage{labels.SplitStartEvent, v, splitOpStart}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return 0, err
}
// Read the sparse volume from reader.
var split dvid.RLEs
split, err = dvid.ReadRLEs(r)
if err != nil {
return
}
toLabelSize, _ := split.Stats()
// Partition the split spans into blocks.
var splitmap dvid.BlockRLEs
splitmap, err = split.Partition(d.BlockSize)
if err != nil {
return
}
// Get a sorted list of blocks that cover split.
splitblks := splitmap.SortedKeys()
// Iterate through the split blocks, read the original block. If the RLEs
// are identical, just delete the original. If not, modify the original.
// TODO: Modifications should be transactional since it's GET-PUT, therefore use
// hash on block coord to direct it to blockLabel, splitLabel-specific goroutine; we serialize
// requests to handle concurrency.
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
for _, splitblk := range splitblks {
// Get original block
tk := NewTKey(fromLabel, splitblk)
val, err := store.Get(ctx, tk)
if err != nil {
return toLabel, err
}
if val == nil {
return toLabel, fmt.Errorf("Split RLEs at block %s are not part of original label %d", splitblk, fromLabel)
}
var rles dvid.RLEs
if err := rles.UnmarshalBinary(val); err != nil {
return toLabel, fmt.Errorf("Unable to unmarshal RLE for original labels in block %s", splitblk)
}
// Compare and process based on modifications required.
remain, err := rles.Split(splitmap[splitblk])
if err != nil {
return toLabel, err
}
if len(remain) == 0 {
batch.Delete(tk)
} else {
rleBytes, err := remain.MarshalBinary()
if err != nil {
return toLabel, fmt.Errorf("can't serialize remain RLEs for split of %d: %v\n", fromLabel, err)
}
batch.Put(tk, rleBytes)
}
}
if err = batch.Commit(); err != nil {
err = fmt.Errorf("Batch PUT during split of %q label %d: %v\n", d.DataName(), fromLabel, err)
return
}
// Publish split event
evt = datastore.SyncEvent{d.DataUUID(), labels.SplitLabelEvent}
msg = datastore.SyncMessage{labels.SplitLabelEvent, v, labels.DeltaSplit{fromLabel, toLabel, splitmap, splitblks}}
if err = datastore.NotifySubscribers(evt, msg); err != nil {
return
}
// Write the split sparse vol.
if err = d.writeLabelVol(v, toLabel, splitmap, splitblks); err != nil {
return
}
// Publish change in label sizes.
delta := labels.DeltaNewSize{
Label: toLabel,
Size: toLabelSize,
}
evt = datastore.SyncEvent{d.DataUUID(), labels.ChangeSizeEvent}
msg = datastore.SyncMessage{labels.ChangeSizeEvent, v, delta}
if err = datastore.NotifySubscribers(evt, msg); err != nil {
return
}
delta2 := labels.DeltaModSize{
Label: fromLabel,
SizeChange: int64(-toLabelSize),
}
evt = datastore.SyncEvent{d.DataUUID(), labels.ChangeSizeEvent}
msg = datastore.SyncMessage{labels.ChangeSizeEvent, v, delta2}
if err = datastore.NotifySubscribers(evt, msg); err != nil {
return
}
// Publish split end
evt = datastore.SyncEvent{d.DataUUID(), labels.SplitEndEvent}
msg = datastore.SyncMessage{labels.SplitEndEvent, v, splitOpEnd}
if err = datastore.NotifySubscribers(evt, msg); err != nil {
return
}
return toLabel, nil
}
// PutSparseVol stores an encoded sparse volume that stays within a given forward label.
// Note that this encoded sparse volume is added to any existing sparse volume for
// a body rather than replace it. To carve out a part of an existing sparse volume,
// use the SplitLabels().
//
// This function handles modification/deletion of all denormalized data touched by this
// sparse label volume.
//
// EVENTS
//
//
func (d *Data) PutSparseVol(v dvid.VersionID, label uint64, r io.Reader) error {
store, err := storage.SmallDataStore()
if err != nil {
return fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
return fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store)
}
// Mark the label as dirty until done.
iv := dvid.InstanceVersion{d.DataName(), v}
dirtyLabels.Incr(iv, label)
defer dirtyLabels.Decr(iv, label)
// TODO -- Signal that we are starting a label modification.
// Read the sparse volume from reader.
header := make([]byte, 8)
if _, err = io.ReadFull(r, header); err != nil {
return err
}
if header[0] != dvid.EncodingBinary {
return fmt.Errorf("sparse vol for split has unknown encoding format: %v", header[0])
}
var numSpans uint32
if err = binary.Read(r, binary.LittleEndian, &numSpans); err != nil {
return err
}
var mods dvid.RLEs
if err = mods.UnmarshalBinaryReader(r, numSpans); err != nil {
return err
}
// Partition the mods spans into blocks.
var modmap dvid.BlockRLEs
modmap, err = mods.Partition(d.BlockSize)
if err != nil {
return err
}
// Publish sparsevol mod event
evt := datastore.SyncEvent{d.DataName(), labels.SparsevolModEvent}
msg := datastore.SyncMessage{v, labels.DeltaSparsevol{label, modmap}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
// Get a sorted list of blocks that cover mods.
modblks := modmap.SortedKeys()
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
var voxelsAdded int64
for _, modblk := range modblks {
// Get original block
tk := NewTKey(label, modblk)
val, err := store.Get(ctx, tk)
if err != nil {
return err
}
// If there's no original block, just write the mod block.
if val == nil || len(val) == 0 {
numVoxels, _ := modmap[modblk].Stats()
voxelsAdded += int64(numVoxels)
rleBytes, err := modmap[modblk].MarshalBinary()
if err != nil {
return fmt.Errorf("can't serialize modified RLEs for %d: %v\n", label, err)
}
batch.Put(tk, rleBytes)
continue
}
// if there's an original, integrate and write merged RLE.
var rles dvid.RLEs
if err := rles.UnmarshalBinary(val); err != nil {
return fmt.Errorf("Unable to unmarshal RLE for original labels in block %s", modblk.Print())
}
voxelsAdded += rles.Add(modmap[modblk])
rleBytes, err := rles.MarshalBinary()
if err != nil {
return fmt.Errorf("can't serialize modified RLEs of %d: %v\n", label, err)
}
batch.Put(tk, rleBytes)
}
if err := batch.Commit(); err != nil {
return fmt.Errorf("Batch commit during mod of %s label %d: %v\n", d.DataName(), label, err)
}
// Publish change in label sizes.
delta := labels.DeltaModSize{
Label: label,
SizeChange: voxelsAdded,
}
evt = datastore.SyncEvent{d.DataName(), labels.ChangeSizeEvent}
msg = datastore.SyncMessage{v, delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
// TODO -- Publish label mod end
return nil
}