func (d *Data) GetKeysInRange(ctx storage.Context, keyBeg, keyEnd string) ([]string, error) {
db, err := storage.BigDataStore()
if err != nil {
return nil, err
}
// Compute first and last key for range
first, err := NewTKey(keyBeg)
if err != nil {
return nil, err
}
last, err := NewTKey(keyEnd)
if err != nil {
return nil, err
}
keys, err := db.KeysInRange(ctx, first, last)
if err != nil {
return nil, err
}
keyList := []string{}
for _, key := range keys {
keyStr, err := DecodeTKey(key)
if err != nil {
return nil, err
}
keyList = append(keyList, keyStr)
}
return keyList, nil
}
// Returns function that stores a tile as an optionally compressed PNG image.
func (d *Data) putTileFunc(versionID dvid.VersionID) (outFunc, error) {
bigdata, err := storage.BigDataStore()
if err != nil {
return nil, fmt.Errorf("Cannot open big data store: %v\n", err)
}
ctx := datastore.NewVersionedCtx(d, versionID)
return func(index *IndexTile, tile *dvid.Image) error {
var err error
var data []byte
switch d.Encoding {
case LZ4:
compression, err := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
if err != nil {
return err
}
data, err = tile.Serialize(compression, d.Checksum())
case PNG:
data, err = tile.GetPNG()
case JPG:
data, err = tile.GetJPEG(d.Quality)
}
if err != nil {
return err
}
return bigdata.Put(ctx, index.Bytes(), data)
}, nil
}
// Loads blocks with old data if they exist.
func (d *Data) loadOldBlocks(v dvid.VersionID, vox *Voxels, blocks storage.TKeyValues) error {
store, err := storage.BigDataStore()
if err != nil {
return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
ctx := datastore.NewVersionedCtx(d, v)
// Create a map of old blocks indexed by the index
oldBlocks := map[dvid.IZYXString]([]byte){}
// Iterate through index space for this data using ZYX ordering.
blockSize := d.BlockSize()
blockNum := 0
for it, err := vox.IndexIterator(blockSize); err == nil && it.Valid(); it.NextSpan() {
indexBeg, indexEnd, err := it.IndexSpan()
if err != nil {
return err
}
begTKey := NewTKey(indexBeg)
endTKey := NewTKey(indexEnd)
// Get previous data.
keyvalues, err := store.GetRange(ctx, begTKey, endTKey)
if err != nil {
return err
}
for _, kv := range keyvalues {
indexZYX, err := DecodeTKey(kv.K)
if err != nil {
return err
}
block, _, err := dvid.DeserializeData(kv.V, true)
if err != nil {
return fmt.Errorf("Unable to deserialize block, %s: %v", ctx, err)
}
oldBlocks[indexZYX.ToIZYXString()] = block
}
// Load previous data into blocks
ptBeg := indexBeg.Duplicate().(dvid.ChunkIndexer)
ptEnd := indexEnd.Duplicate().(dvid.ChunkIndexer)
begX := ptBeg.Value(0)
endX := ptEnd.Value(0)
c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)}
for x := begX; x <= endX; x++ {
c[0] = x
curIndex := dvid.IndexZYX(c)
curTKey := NewTKey(&curIndex)
blocks[blockNum].K = curTKey
block, ok := oldBlocks[curIndex.ToIZYXString()]
if ok {
copy(blocks[blockNum].V, block)
}
blockNum++
}
}
return nil
}
// GetVoxels copies voxels from the storage engine to Voxels, a requested subvolume or 2d image.
func (d *Data) GetVoxels(v dvid.VersionID, vox *Voxels, r *ROI) error {
timedLog := dvid.NewTimeLog()
defer timedLog.Infof("GetVoxels %s", vox)
store, err := storage.BigDataStore()
if err != nil {
return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
// Only do one request at a time, although each request can start many goroutines.
server.SpawnGoroutineMutex.Lock()
defer server.SpawnGoroutineMutex.Unlock()
ctx := datastore.NewVersionedCtx(d, v)
wg := new(sync.WaitGroup)
for it, err := vox.IndexIterator(d.BlockSize()); err == nil && it.Valid(); it.NextSpan() {
indexBeg, indexEnd, err := it.IndexSpan()
if err != nil {
return err
}
begTKey := NewTKey(indexBeg)
endTKey := NewTKey(indexEnd)
// Get set of blocks in ROI if ROI provided
var chunkOp *storage.ChunkOp
if r != nil && r.Iter != nil {
ptBeg := indexBeg.Duplicate().(dvid.ChunkIndexer)
ptEnd := indexEnd.Duplicate().(dvid.ChunkIndexer)
begX := ptBeg.Value(0)
endX := ptEnd.Value(0)
blocksInROI := make(map[string]bool, (endX - begX + 1))
c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)}
for x := begX; x <= endX; x++ {
c[0] = x
curIndex := dvid.IndexZYX(c)
if r.Iter.InsideFast(curIndex) {
indexString := string(curIndex.Bytes())
blocksInROI[indexString] = true
}
}
chunkOp = &storage.ChunkOp{&getOperation{vox, blocksInROI, r.attenuation}, wg}
} else {
chunkOp = &storage.ChunkOp{&getOperation{vox, nil, 0}, wg}
}
// Send the entire range of key-value pairs to chunk processor
err = store.ProcessRange(ctx, begTKey, endTKey, chunkOp, storage.ChunkFunc(d.ReadChunk))
if err != nil {
return fmt.Errorf("Unable to GET data %s: %v", ctx, err)
}
}
if err != nil {
return err
}
wg.Wait()
return nil
}
// GetBlocks returns a slice of bytes corresponding to all the blocks along a span in X
func (d *Data) GetBlocks(v dvid.VersionID, start dvid.ChunkPoint3d, span int) ([]byte, error) {
store, err := storage.BigDataStore()
if err != nil {
return nil, fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
indexBeg := dvid.IndexZYX(start)
end := start
end[0] += int32(span - 1)
indexEnd := dvid.IndexZYX(end)
begTKey := NewTKey(&indexBeg)
endTKey := NewTKey(&indexEnd)
ctx := datastore.NewVersionedCtx(d, v)
iv := dvid.InstanceVersion{d.DataName(), v}
mapping := labels.MergeCache.LabelMap(iv)
keyvalues, err := store.GetRange(ctx, begTKey, endTKey)
if err != nil {
return nil, err
}
var buf bytes.Buffer
// Save the # of keyvalues actually obtained.
numkv := len(keyvalues)
binary.Write(&buf, binary.LittleEndian, int32(numkv))
// Write the block indices in XYZ little-endian format + the size of each block
uncompress := true
for _, kv := range keyvalues {
block, _, err := dvid.DeserializeData(kv.V, uncompress)
if err != nil {
return nil, fmt.Errorf("Unable to deserialize block, %s (%v): %v", ctx, kv.K, err)
}
if mapping != nil {
n := len(block) / 8
for i := 0; i < n; i++ {
orig := binary.LittleEndian.Uint64(block[i*8 : i*8+8])
mapped, found := mapping.FinalLabel(orig)
if !found {
mapped = orig
}
binary.LittleEndian.PutUint64(block[i*8:i*8+8], mapped)
}
}
_, err = buf.Write(block)
if err != nil {
return nil, err
}
}
return buf.Bytes(), nil
}
// Goroutine that handles relabeling of blocks during a merge operation.
// Since the same block coordinate always gets mapped to the same goroutine, we handle
// concurrency by serializing GET/PUT for a particular block coordinate.
func (d *Data) mergeBlock(in <-chan mergeOp) {
store, err := storage.BigDataStore()
if err != nil {
dvid.Errorf("Data type labelblk had error initializing store: %v\n", err)
return
}
blockBytes := int(d.BlockSize().Prod() * 8)
for op := range in {
tk := NewTKeyByCoord(op.izyx)
data, err := store.Get(op.ctx, tk)
if err != nil {
dvid.Errorf("Error on GET of labelblk with coord string %q\n", op.izyx)
op.wg.Done()
continue
}
if data == nil {
dvid.Errorf("nil label block where merge was done!\n")
op.wg.Done()
continue
}
blockData, _, err := dvid.DeserializeData(data, true)
if err != nil {
dvid.Criticalf("unable to deserialize label block in '%s': %v\n", d.DataName(), err)
op.wg.Done()
continue
}
if len(blockData) != blockBytes {
dvid.Criticalf("After labelblk deserialization got back %d bytes, expected %d bytes\n", len(blockData), blockBytes)
op.wg.Done()
continue
}
// Iterate through this block of labels and relabel if label in merge.
for i := 0; i < blockBytes; i += 8 {
label := binary.LittleEndian.Uint64(blockData[i : i+8])
if _, merged := op.Merged[label]; merged {
binary.LittleEndian.PutUint64(blockData[i:i+8], op.Target)
}
}
// Store this block.
serialization, err := dvid.SerializeData(blockData, d.Compression(), d.Checksum())
if err != nil {
dvid.Criticalf("Unable to serialize block in %q: %v\n", d.DataName(), err)
op.wg.Done()
continue
}
if err := store.Put(op.ctx, tk, serialization); err != nil {
dvid.Errorf("Error in putting key %v: %v\n", tk, err)
}
op.wg.Done()
}
}
// DeleteData deletes a key-value pair
func (d *Data) DeleteData(ctx storage.Context, keyStr string) error {
db, err := storage.BigDataStore()
if err != nil {
return err
}
tk, err := NewTKey(keyStr)
if err != nil {
return err
}
return db.Delete(ctx, tk)
}
// PutData puts a key-value at a given uuid
func (d *Data) PutData(ctx storage.Context, keyStr string, value []byte) error {
db, err := storage.BigDataStore()
if err != nil {
return err
}
serialization, err := dvid.SerializeData(value, d.Compression(), d.Checksum())
if err != nil {
return fmt.Errorf("Unable to serialize data: %v\n", err)
}
tk, err := NewTKey(keyStr)
if err != nil {
return err
}
return db.Put(ctx, tk, serialization)
}
// getTileData returns 2d tile data straight from storage without decoding.
func (d *Data) getTileData(ctx storage.Context, shape dvid.DataShape, scaling Scaling, index dvid.IndexZYX) ([]byte, error) {
if d.Levels == nil {
return nil, fmt.Errorf("Tiles have not been generated.")
}
bigdata, err := storage.BigDataStore()
if err != nil {
return nil, err
}
// Retrieve the tile data from datastore
tileIndex := &IndexTile{&index, shape, scaling}
data, err := bigdata.Get(ctx, tileIndex.Bytes())
if err != nil {
return nil, fmt.Errorf("Error trying to GET from datastore: %v", err)
}
return data, nil
}
// GetSurface returns a gzipped byte array with # voxels and float32 arrays for vertices and
// normals.
func GetSurface(ctx storage.Context, label uint64) ([]byte, bool, error) {
bigdata, err := storage.BigDataStore()
if err != nil {
return nil, false, fmt.Errorf("Cannot get datastore that handles big data: %v\n", err)
}
// Retrieve the precomputed surface or that it's not available.
data, err := bigdata.Get(ctx, NewLabelSurfaceIndex(label))
if err != nil {
return nil, false, fmt.Errorf("Error in retrieving surface for label %d: %v", label, err)
}
if data == nil {
return []byte{}, false, nil
}
uncompress := false
surfaceBytes, _, err := dvid.DeserializeData(data, uncompress)
if err != nil {
return nil, false, fmt.Errorf("Unable to deserialize surface for label %d: %v\n", label, err)
}
return surfaceBytes, true, nil
}
// GetData gets a value using a key
func (d *Data) GetData(ctx storage.Context, keyStr string) ([]byte, bool, error) {
db, err := storage.BigDataStore()
if err != nil {
return nil, false, err
}
tk, err := NewTKey(keyStr)
if err != nil {
return nil, false, err
}
data, err := db.Get(ctx, tk)
if err != nil {
return nil, false, fmt.Errorf("Error in retrieving key '%s': %v", keyStr, err)
}
if data == nil {
return nil, false, nil
}
uncompress := true
value, _, err := dvid.DeserializeData(data, uncompress)
if err != nil {
return nil, false, fmt.Errorf("Unable to deserialize data for key '%s': %v\n", keyStr, err)
}
return value, true, nil
}
// GetLabelBytesAtPoint returns the 8 byte slice corresponding to a 64-bit label at a point.
func (d *Data) GetLabelBytesAtPoint(v dvid.VersionID, pt dvid.Point) ([]byte, error) {
store, err := storage.BigDataStore()
if err != nil {
return nil, err
}
// Compute the block key that contains the given point.
coord, ok := pt.(dvid.Chunkable)
if !ok {
return nil, fmt.Errorf("Can't determine block of point %s", pt)
}
blockSize := d.BlockSize()
blockCoord := coord.Chunk(blockSize).(dvid.ChunkPoint3d)
index := dvid.IndexZYX(blockCoord)
// Retrieve the block of labels
ctx := datastore.NewVersionedCtx(d, v)
serialization, err := store.Get(ctx, NewTKey(&index))
if err != nil {
return nil, fmt.Errorf("Error getting '%s' block for index %s\n", d.DataName(), blockCoord)
}
if serialization == nil {
return zeroLabelBytes, nil
}
labelData, _, err := dvid.DeserializeData(serialization, true)
if err != nil {
return nil, fmt.Errorf("Unable to deserialize block %s in '%s': %v\n", blockCoord, d.DataName(), err)
}
// Retrieve the particular label within the block.
ptInBlock := coord.PointInChunk(blockSize)
nx := int64(blockSize.Value(0))
nxy := nx * int64(blockSize.Value(1))
i := (int64(ptInBlock.Value(0)) + int64(ptInBlock.Value(1))*nx + int64(ptInBlock.Value(2))*nxy) * 8
return labelData[i : i+8], nil
}
func (d *Data) createCompositeChunk(chunk *storage.Chunk) {
defer func() {
// After processing a chunk, return the token.
server.HandlerToken <- 1
// Notify the requestor that this chunk is done.
if chunk.Wg != nil {
chunk.Wg.Done()
}
}()
op := chunk.Op.(*blockOp)
// Get the spatial index associated with this chunk.
zyx, err := imageblk.DecodeTKey(chunk.K)
if err != nil {
dvid.Errorf("Error in %s.ChunkApplyMap(): %v", d.Data.DataName(), err)
return
}
// Initialize the label buffers. For voxels, this data needs to be uncompressed and deserialized.
curZMutex.Lock()
if zyx[2] > curZ {
curZ = zyx[2]
minZ := zyx.MinPoint(d.BlockSize()).Value(2)
maxZ := zyx.MaxPoint(d.BlockSize()).Value(2)
dvid.Debugf("Now creating composite blocks for Z %d to %d\n", minZ, maxZ)
}
curZMutex.Unlock()
labelData, _, err := dvid.DeserializeData(chunk.V, true)
if err != nil {
dvid.Infof("Unable to deserialize block in '%s': %v\n", d.DataName(), err)
return
}
blockBytes := len(labelData)
if blockBytes%8 != 0 {
dvid.Infof("Retrieved, deserialized block is wrong size: %d bytes\n", blockBytes)
return
}
// Get the corresponding grayscale block.
bigdata, err := storage.BigDataStore()
if err != nil {
dvid.Errorf("Unable to retrieve big data store: %s\n", err)
return
}
grayscaleCtx := datastore.NewVersionedCtx(op.grayscale, op.versionID)
blockData, err := bigdata.Get(grayscaleCtx, chunk.K)
if err != nil {
dvid.Errorf("Error getting grayscale block for index %s\n", zyx)
return
}
grayscaleData, _, err := dvid.DeserializeData(blockData, true)
if err != nil {
dvid.Errorf("Unable to deserialize block in '%s': %v\n", op.grayscale.DataName(), err)
return
}
// Compute the composite block.
// TODO -- Exploit run lengths, use cache of hash?
compositeBytes := blockBytes / 2
compositeData := make([]byte, compositeBytes, compositeBytes)
compositeI := 0
labelI := 0
hashBuf := make([]byte, 4, 4)
for _, grayscale := range grayscaleData {
//murmurhash3(labelData[labelI:labelI+8], hashBuf)
//hashBuf[3] = grayscale
writePseudoColor(grayscale, labelData[labelI:labelI+8], hashBuf)
copy(compositeData[compositeI:compositeI+4], hashBuf)
compositeI += 4
labelI += 8
}
// Store the composite block into the rgba8 data.
serialization, err := dvid.SerializeData(compositeData, d.Compression(), d.Checksum())
if err != nil {
dvid.Errorf("Unable to serialize composite block %s: %v\n", zyx, err)
return
}
compositeCtx := datastore.NewVersionedCtx(op.composite, op.versionID)
err = bigdata.Put(compositeCtx, chunk.K, serialization)
if err != nil {
dvid.Errorf("Unable to PUT composite block %s: %v\n", zyx, err)
return
}
}
// Calculates value of a 3d real world point in space defined by underlying data resolution.
func (d *Data) computeValue(pt dvid.Vector3d, ctx storage.Context, keyF KeyFunc, cache *ValueCache) ([]byte, error) {
db, err := storage.BigDataStore()
if err != nil {
return nil, err
}
valuesPerElement := d.Properties.Values.ValuesPerElement()
bytesPerValue, err := d.Properties.Values.BytesPerValue()
if err != nil {
return nil, err
}
bytesPerVoxel := valuesPerElement * bytesPerValue
// Allocate an empty block.
blockSize, ok := d.BlockSize().(dvid.Point3d)
if !ok {
return nil, fmt.Errorf("Data %q does not have a 3d block size", d.DataName())
}
nx := blockSize[0]
nxy := nx * blockSize[1]
emptyBlock := d.BackgroundBlock()
populateF := func(key []byte) ([]byte, error) {
serializedData, err := db.Get(ctx, key)
if err != nil {
return nil, err
}
var deserializedData []byte
if serializedData == nil || len(serializedData) == 0 {
deserializedData = emptyBlock
} else {
deserializedData, _, err = dvid.DeserializeData(serializedData, true)
if err != nil {
return nil, fmt.Errorf("Unable to deserialize block: %v", err)
}
}
return deserializedData, nil
}
// For the given point, compute surrounding lattice points and retrieve values.
neighbors := d.neighborhood(pt)
var valuesI int32
for _, voxelCoord := range neighbors.coords {
deserializedData, _, err := cache.Get(keyF(voxelCoord), populateF)
if err != nil {
return nil, err
}
blockPt := voxelCoord.PointInChunk(blockSize).(dvid.Point3d)
blockI := blockPt[2]*nxy + blockPt[1]*nx + blockPt[0]
//fmt.Printf("Block %s (%d) len %d -> Neighbor %s (buffer %d, len %d)\n",
// blockPt, blockI, len(blockData), voxelCoord, valuesI, len(neighbors.values))
copy(neighbors.values[valuesI:valuesI+bytesPerVoxel], deserializedData[blockI:blockI+bytesPerVoxel])
valuesI += bytesPerVoxel
}
// Perform trilinear interpolation on the underlying data values.
unsupported := func() error {
return fmt.Errorf("DVID cannot retrieve images with arbitrary orientation using %d channels and %d bytes/channel",
valuesPerElement, bytesPerValue)
}
var value []byte
switch valuesPerElement {
case 1:
switch bytesPerValue {
case 1:
if d.Interpolable {
interpValue := trilinearInterpUint8(neighbors.xd, neighbors.yd, neighbors.zd, []uint8(neighbors.values))
value = []byte{byte(interpValue)}
} else {
value = []byte{nearestNeighborUint8(neighbors.xd, neighbors.yd, neighbors.zd, []uint8(neighbors.values))}
}
case 2:
fallthrough
case 4:
fallthrough
case 8:
fallthrough
default:
return nil, unsupported()
}
case 4:
switch bytesPerValue {
case 1:
value = make([]byte, 4, 4)
for c := 0; c < 4; c++ {
channelValues := make([]uint8, 8, 8)
for i := 0; i < 8; i++ {
channelValues[i] = uint8(neighbors.values[i*4+c])
}
if d.Interpolable {
interpValue := trilinearInterpUint8(neighbors.xd, neighbors.yd, neighbors.zd, channelValues)
value[c] = byte(interpValue)
} else {
value[c] = byte(nearestNeighborUint8(neighbors.xd, neighbors.yd, neighbors.zd, channelValues))
}
}
case 2:
fallthrough
default:
return nil, unsupported()
}
default:
}
return value, nil
}
func (d *Data) foregroundROI(v dvid.VersionID, dest *roi.Data, background dvid.PointNd) {
dest.Ready = false
store, err := storage.BigDataStore()
if err != nil {
dvid.Criticalf("Data type imageblk had error initializing store: %v\n", err)
return
}
timedLog := dvid.NewTimeLog()
timedLog.Infof("Starting foreground ROI %q for %s", dest.DataName(), d.DataName())
// Iterate through all voxel blocks, loading and then checking blocks
// for any foreground voxels.
ctx := datastore.NewVersionedCtx(d, v)
backgroundBytes := make([]byte, len(background))
for i, b := range background {
backgroundBytes[i] = byte(b)
}
const BATCH_SIZE = 1000
var numBatches int
var span *dvid.Span
spans := []dvid.Span{}
var f storage.ChunkFunc = func(chunk *storage.Chunk) error {
if chunk == nil || chunk.V == nil {
return nil
}
data, _, err := dvid.DeserializeData(chunk.V, true)
if err != nil {
return fmt.Errorf("Error decoding block: %v\n", err)
}
numVoxels := d.BlockSize().Prod()
var foreground bool
for i := int64(0); i < numVoxels; i++ {
isBackground := false
for _, b := range backgroundBytes {
if data[i] == b {
isBackground = true
break
}
}
if !isBackground {
foreground = true
break
}
}
if foreground {
indexZYX, err := DecodeTKey(chunk.K)
if err != nil {
return fmt.Errorf("Error decoding voxel block key: %v\n", err)
}
x, y, z := indexZYX.Unpack()
if span == nil {
span = &dvid.Span{z, y, x, x}
} else if !span.Extends(x, y, z) {
spans = append(spans, *span)
if len(spans) >= BATCH_SIZE {
init := (numBatches == 0)
numBatches++
go func(spans []dvid.Span) {
if err := dest.PutSpans(v, spans, init); err != nil {
dvid.Errorf("Error in storing ROI: %v\n", err)
} else {
timedLog.Debugf("-- Wrote batch %d of spans for foreground ROI %q", numBatches, dest.DataName())
}
}(spans)
spans = []dvid.Span{}
}
span = &dvid.Span{z, y, x, x}
}
}
server.BlockOnInteractiveRequests("voxels [compute foreground ROI]")
return nil
}
minTKey := storage.MinTKey(keyImageBlock)
maxTKey := storage.MaxTKey(keyImageBlock)
err = store.ProcessRange(ctx, minTKey, maxTKey, &storage.ChunkOp{}, f)
if err != nil {
dvid.Errorf("Error in processing chunks in ROI: %v\n", err)
return
}
if span != nil {
spans = append(spans, *span)
}
// Save new ROI
if len(spans) > 0 {
if err := dest.PutSpans(v, spans, numBatches == 0); err != nil {
dvid.Errorf("Error in storing ROI: %v\n", err)
return
}
}
timedLog.Infof("Created foreground ROI %q for %s", dest.DataName(), d.DataName())
dest.Ready = true
}
// Goroutine that handles splits across a lot of blocks for one label.
func (d *Data) splitBlock(in <-chan splitOp) {
store, err := storage.BigDataStore()
if err != nil {
dvid.Errorf("Data type labelblk had error initializing store: %v\n", err)
return
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
err = fmt.Errorf("Data type labelblk requires batch-enabled store, which %q is not\n", store)
return
}
blockBytes := int(d.BlockSize().Prod() * 8)
for op := range in {
// Iterate through all the modified blocks, inserting the new label using the RLEs for that block.
timedLog := dvid.NewTimeLog()
splitCache.Incr(op.ctx.InstanceVersion(), op.OldLabel)
batch := batcher.NewBatch(&op.ctx)
for _, zyxStr := range op.SortedBlocks {
// Read the block.
tk := NewTKeyByCoord(zyxStr)
data, err := store.Get(&op.ctx, tk)
if err != nil {
dvid.Errorf("Error on GET of labelblk with coord string %v\n", []byte(zyxStr))
continue
}
if data == nil {
dvid.Errorf("nil label block where split was done, coord %v\n", []byte(zyxStr))
continue
}
bdata, _, err := dvid.DeserializeData(data, true)
if err != nil {
dvid.Criticalf("unable to deserialize label block in '%s' key %v: %v\n", d.DataName(), []byte(zyxStr), err)
continue
}
if len(bdata) != blockBytes {
dvid.Criticalf("splitBlock: coord %v got back %d bytes, expected %d bytes\n", []byte(zyxStr), len(bdata), blockBytes)
continue
}
// Modify the block.
rles, found := op.Split[zyxStr]
if !found {
dvid.Errorf("split block %v not present in block RLEs\n", []byte(zyxStr))
continue
}
if err := d.storeRLEs(zyxStr, bdata, op.NewLabel, rles); err != nil {
dvid.Errorf("can't store RLEs into block %v\n", []byte(zyxStr))
continue
}
// Write the modified block.
serialization, err := dvid.SerializeData(bdata, d.Compression(), d.Checksum())
if err != nil {
dvid.Criticalf("Unable to serialize block in %q: %v\n", d.DataName(), err)
continue
}
batch.Put(tk, serialization)
}
if err := batch.Commit(); err != nil {
dvid.Errorf("Batch PUT during %q block split of %d: %v\n", d.DataName(), op.OldLabel, err)
}
splitCache.Decr(op.ctx.InstanceVersion(), op.OldLabel)
timedLog.Debugf("labelblk sync complete for split of %d -> %d", op.OldLabel, op.NewLabel)
}
}
// GetBlocks returns a slice of bytes corresponding to all the blocks along a span in X
func (d *Data) GetBlocks(v dvid.VersionID, start dvid.ChunkPoint3d, span int32) ([]byte, error) {
timedLog := dvid.NewTimeLog()
defer timedLog.Infof("GetBlocks %s, span %d", start, span)
store, err := storage.BigDataStore()
if err != nil {
return nil, fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
indexBeg := dvid.IndexZYX(start)
sx, sy, sz := indexBeg.Unpack()
end := start
end[0] += int32(span - 1)
indexEnd := dvid.IndexZYX(end)
keyBeg := NewTKey(&indexBeg)
keyEnd := NewTKey(&indexEnd)
// Allocate one uncompressed-sized slice with background values.
blockBytes := int32(d.BlockSize().Prod()) * d.Values.BytesPerElement()
numBytes := blockBytes * span
buf := make([]byte, numBytes, numBytes)
if d.Background != 0 {
for i := range buf {
buf[i] = byte(d.Background)
}
}
// Write the blocks that we can get concurrently on this byte slice.
ctx := datastore.NewVersionedCtx(d, v)
var wg sync.WaitGroup
err = store.ProcessRange(ctx, keyBeg, keyEnd, &storage.ChunkOp{}, func(c *storage.Chunk) error {
if c == nil || c.TKeyValue == nil {
return nil
}
kv := c.TKeyValue
if kv.V == nil {
return nil
}
// Determine which block this is.
indexZYX, err := DecodeTKey(kv.K)
if err != nil {
return err
}
x, y, z := indexZYX.Unpack()
if z != sz || y != sy || x < sx || x >= sx+int32(span) {
return fmt.Errorf("Received key-value for %s, not supposed to be within span range %s, length %d", *indexZYX, start, span)
}
n := x - sx
i := n * blockBytes
j := i + blockBytes
// Spawn goroutine to transfer data
wg.Add(1)
go xferBlock(buf[i:j], c, &wg)
return nil
})
if err != nil {
return nil, err
}
wg.Wait()
return buf, nil
}
// TODO -- Clean up all the writing and simplify now that we have block-aligned writes.
// writeBlocks persists blocks of voxel data asynchronously using batch writes.
func (d *Data) writeBlocks(v dvid.VersionID, b storage.TKeyValues, wg1, wg2 *sync.WaitGroup) error {
store, err := storage.BigDataStore()
if err != nil {
return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
return fmt.Errorf("Data type imageblk requires batch-enabled store, which %q is not\n", store)
}
preCompress, postCompress := 0, 0
ctx := datastore.NewVersionedCtx(d, v)
evt := datastore.SyncEvent{d.DataName(), ChangeBlockEvent}
<-server.HandlerToken
go func() {
defer func() {
wg1.Done()
wg2.Done()
dvid.Debugf("Wrote voxel blocks. Before %s: %d bytes. After: %d bytes\n", d.Compression(), preCompress, postCompress)
server.HandlerToken <- 1
}()
batch := batcher.NewBatch(ctx)
for i, block := range b {
serialization, err := dvid.SerializeData(block.V, d.Compression(), d.Checksum())
preCompress += len(block.V)
postCompress += len(serialization)
if err != nil {
dvid.Errorf("Unable to serialize block: %v\n", err)
return
}
batch.Put(block.K, serialization)
indexZYX, err := DecodeTKey(block.K)
if err != nil {
dvid.Errorf("Unable to recover index from block key: %v\n", block.K)
return
}
msg := datastore.SyncMessage{v, Block{indexZYX, block.V}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Errorf("Unable to notify subscribers of ChangeBlockEvent in %s\n", d.DataName())
return
}
// Check if we should commit
if i%KVWriteSize == KVWriteSize-1 {
if err := batch.Commit(); err != nil {
dvid.Errorf("Error on trying to write batch: %v\n", err)
return
}
batch = batcher.NewBatch(ctx)
}
}
if err := batch.Commit(); err != nil {
dvid.Errorf("Error on trying to write batch: %v\n", err)
return
}
}()
return nil
}
func (d *Data) putChunk(chunk *storage.Chunk) {
defer func() {
// After processing a chunk, return the token.
server.HandlerToken <- 1
// Notify the requestor that this chunk is done.
if chunk.Wg != nil {
chunk.Wg.Done()
}
}()
op, ok := chunk.Op.(*putOperation)
if !ok {
log.Fatalf("Illegal operation passed to ProcessChunk() for data %s\n", d.DataName())
}
// Make sure our received chunk is valid.
if chunk == nil {
dvid.Errorf("Received nil chunk in ProcessChunk. Ignoring chunk.\n")
return
}
if chunk.K == nil {
dvid.Errorf("Received nil chunk key in ProcessChunk. Ignoring chunk.\n")
return
}
// Initialize the block buffer using the chunk of data. For voxels, this chunk of
// data needs to be uncompressed and deserialized.
var blockData []byte
var err error
if chunk.V == nil {
blockData = d.BackgroundBlock()
} else {
blockData, _, err = dvid.DeserializeData(chunk.V, true)
if err != nil {
dvid.Errorf("Unable to deserialize block in '%s': %v\n", d.DataName(), err)
return
}
}
// Perform the operation.
block := &storage.TKeyValue{K: chunk.K, V: blockData}
if err = op.voxels.WriteBlock(block, d.BlockSize()); err != nil {
dvid.Errorf("Unable to WriteBlock() in %q: %v\n", d.DataName(), err)
return
}
serialization, err := dvid.SerializeData(blockData, d.Compression(), d.Checksum())
if err != nil {
dvid.Errorf("Unable to serialize block in %q: %v\n", d.DataName(), err)
return
}
store, err := storage.BigDataStore()
if err != nil {
dvid.Errorf("Data type imageblk had error initializing store: %v\n", err)
return
}
ctx := datastore.NewVersionedCtx(d, op.version)
if err := store.Put(ctx, chunk.K, serialization); err != nil {
dvid.Errorf("Unable to PUT voxel data for key %v: %v\n", chunk.K, err)
return
}
// Notify any subscribers that you've changed block.
evt := datastore.SyncEvent{d.DataName(), ChangeBlockEvent}
msg := datastore.SyncMessage{op.version, Block{&op.indexZYX, block.V}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Errorf("Unable to notify subscribers of ChangeBlockEvent in %s\n", d.DataName())
}
}
// PutBlocks stores blocks of data in a span along X
func (d *Data) PutBlocks(v dvid.VersionID, start dvid.ChunkPoint3d, span int, data io.ReadCloser) error {
store, err := storage.BigDataStore()
if err != nil {
return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
return fmt.Errorf("Data type imageblk requires batch-enabled store, which %q is not\n", store)
}
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
// Read blocks from the stream until we can output a batch put.
const BatchSize = 1000
var readBlocks int
numBlockBytes := d.BlockSize().Prod()
chunkPt := start
buf := make([]byte, numBlockBytes)
for {
// Read a block's worth of data
readBytes := int64(0)
for {
n, err := data.Read(buf[readBytes:])
readBytes += int64(n)
if readBytes == numBlockBytes {
break
}
if err == io.EOF {
return fmt.Errorf("Block data ceased before all block data read")
}
if err != nil {
return fmt.Errorf("Error reading blocks: %v\n", err)
}
}
if readBytes != numBlockBytes {
return fmt.Errorf("Expected %d bytes in block read, got %d instead! Aborting.", numBlockBytes, readBytes)
}
serialization, err := dvid.SerializeData(buf, d.Compression(), d.Checksum())
if err != nil {
return err
}
zyx := dvid.IndexZYX(chunkPt)
batch.Put(NewTKey(&zyx), serialization)
// Notify any subscribers that you've changed block.
evt := datastore.SyncEvent{d.DataName(), ChangeBlockEvent}
msg := datastore.SyncMessage{v, Block{&zyx, buf}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
// Advance to next block
chunkPt[0]++
readBlocks++
finish := (readBlocks == span)
if finish || readBlocks%BatchSize == 0 {
if err := batch.Commit(); err != nil {
return fmt.Errorf("Error on batch commit, block %d: %v\n", readBlocks, err)
}
batch = batcher.NewBatch(ctx)
}
if finish {
break
}
}
return nil
}
// CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale
func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error {
timedLog := dvid.NewTimeLog()
// Parse the request
var uuidStr, dataName, cmdStr, grayscaleName, destName string
request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName)
// Get the version
uuid, v, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
// Log request
if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
return err
}
// Get the grayscale data.
dataservice, err := datastore.GetDataByUUID(uuid, dvid.InstanceName(grayscaleName))
if err != nil {
return err
}
grayscale, ok := dataservice.(*imageblk.Data)
if !ok {
return fmt.Errorf("%s is not the name of uint8 data", grayscaleName)
}
// Create a new rgba8blk data.
var compservice datastore.DataService
compservice, err = datastore.GetDataByUUID(uuid, dvid.InstanceName(destName))
if err == nil {
return fmt.Errorf("Data instance with name %q already exists", destName)
}
typeService, err := datastore.TypeServiceByName("rgba8blk")
if err != nil {
return fmt.Errorf("Could not get rgba8 type service from DVID")
}
config := dvid.NewConfig()
compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config)
if err != nil {
return err
}
composite, ok := compservice.(*imageblk.Data)
if !ok {
return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName)
}
// Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time.
wg := new(sync.WaitGroup)
op := &blockOp{grayscale, composite, v}
chunkOp := &storage.ChunkOp{op, wg}
store, err := storage.BigDataStore()
if err != nil {
return err
}
ctx := datastore.NewVersionedCtx(d, v)
extents := d.Extents()
blockBeg := imageblk.NewTKey(extents.MinIndex)
blockEnd := imageblk.NewTKey(extents.MaxIndex)
err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk))
wg.Wait()
// Set new mapped data to same extents.
composite.Properties.Extents = grayscale.Properties.Extents
if err := datastore.SaveDataByUUID(uuid, composite); err != nil {
dvid.Infof("Could not save new data '%s': %v\n", destName, err)
}
timedLog.Infof("Created composite of %s and %s", grayscaleName, destName)
return nil
}
func (d *Data) DeleteBlocks(ctx *datastore.VersionedCtx, start dvid.ChunkPoint3d, span int) error {
store, err := storage.BigDataStore()
if err != nil {
return fmt.Errorf("Data type labelblk had error initializing store: %v\n", err)
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
return fmt.Errorf("Data type labelblk requires batch-enabled store, which %q is not\n", store)
}
indexBeg := dvid.IndexZYX(start)
end := start
end[0] += int32(span - 1)
indexEnd := dvid.IndexZYX(end)
begTKey := NewTKey(&indexBeg)
endTKey := NewTKey(&indexEnd)
iv := dvid.InstanceVersion{d.DataName(), ctx.VersionID()}
mapping := labels.MergeCache.LabelMap(iv)
kvs, err := store.GetRange(ctx, begTKey, endTKey)
if err != nil {
return err
}
batch := batcher.NewBatch(ctx)
uncompress := true
for _, kv := range kvs {
izyx, err := DecodeTKey(kv.K)
if err != nil {
return err
}
// Delete the labelblk (really tombstones it)
batch.Delete(kv.K)
// Send data to delete associated labelvol for labels in this block
block, _, err := dvid.DeserializeData(kv.V, uncompress)
if err != nil {
return fmt.Errorf("Unable to deserialize block, %s (%v): %v", ctx, kv.K, err)
}
if mapping != nil {
n := len(block) / 8
for i := 0; i < n; i++ {
orig := binary.LittleEndian.Uint64(block[i*8 : i*8+8])
mapped, found := mapping.FinalLabel(orig)
if !found {
mapped = orig
}
binary.LittleEndian.PutUint64(block[i*8:i*8+8], mapped)
}
}
// Notify any subscribers that we've deleted this block.
evt := datastore.SyncEvent{d.DataName(), labels.DeleteBlockEvent}
msg := datastore.SyncMessage{ctx.VersionID(), labels.DeleteBlock{izyx, block}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
}
return batch.Commit()
}