// Goroutine that handles splits across a lot of blocks for one label.
func (d *Data) splitBlock(ctx *datastore.VersionedCtx, op splitOp) {
defer d.MutDone(op.mutID)
store, err := d.GetOrderedKeyValueDB()
if err != nil {
dvid.Errorf("Data type labelblk had error initializing store: %v\n", err)
return
}
// Read the block.
tk := NewTKeyByCoord(op.block)
data, err := store.Get(ctx, tk)
if err != nil {
dvid.Errorf("Error on GET of labelblk with coord string %v\n", []byte(op.block))
return
}
if data == nil {
dvid.Errorf("nil label block where split was done, coord %v\n", []byte(op.block))
return
}
blockData, _, err := dvid.DeserializeData(data, true)
if err != nil {
dvid.Criticalf("unable to deserialize label block in '%s' key %v: %v\n", d.DataName(), []byte(op.block), err)
return
}
blockBytes := int(d.BlockSize().Prod() * 8)
if len(blockData) != blockBytes {
dvid.Criticalf("splitBlock: coord %v got back %d bytes, expected %d bytes\n", []byte(op.block), len(blockData), blockBytes)
return
}
// Modify the block using either voxel-level changes or coarser block-level mods.
if op.rles != nil {
if err := d.storeRLEs(blockData, op.newLabel, op.block, op.rles); err != nil {
dvid.Errorf("can't store label %d RLEs into block %s: %v\n", op.newLabel, op.block, err)
return
}
} else {
// We are doing coarse split and will replace all
if err := d.replaceLabel(blockData, op.oldLabel, op.newLabel); err != nil {
dvid.Errorf("can't replace label %d with %d in block %s: %v\n", op.oldLabel, op.newLabel, op.block, err)
return
}
}
// Write the modified block.
serialization, err := dvid.SerializeData(blockData, d.Compression(), d.Checksum())
if err != nil {
dvid.Criticalf("Unable to serialize block %s in %q: %v\n", op.block, d.DataName(), err)
return
}
if err := store.Put(ctx, tk, serialization); err != nil {
dvid.Errorf("Error in putting key %v: %v\n", tk, err)
}
// Notify any downstream downres instance.
d.publishBlockChange(ctx.VersionID(), op.mutID, op.block, blockData)
}
// handles relabeling of blocks during a merge operation.
func (d *Data) mergeBlock(ctx *datastore.VersionedCtx, op mergeOp) {
defer d.MutDone(op.mutID)
store, err := d.GetKeyValueDB()
if err != nil {
dvid.Errorf("Data type labelblk had error initializing store: %v\n", err)
return
}
tk := NewTKeyByCoord(op.block)
data, err := store.Get(ctx, tk)
if err != nil {
dvid.Errorf("Error on GET of labelblk with coord string %q\n", op.block)
return
}
if data == nil {
dvid.Errorf("nil label block where merge was done!\n")
return
}
blockData, _, err := dvid.DeserializeData(data, true)
if err != nil {
dvid.Criticalf("unable to deserialize label block in '%s': %v\n", d.DataName(), err)
return
}
blockBytes := int(d.BlockSize().Prod() * 8)
if len(blockData) != blockBytes {
dvid.Criticalf("After labelblk deserialization got back %d bytes, expected %d bytes\n", len(blockData), blockBytes)
return
}
// Iterate through this block of labels and relabel if label in merge.
for i := 0; i < blockBytes; i += 8 {
label := binary.LittleEndian.Uint64(blockData[i : i+8])
if _, merged := op.Merged[label]; merged {
binary.LittleEndian.PutUint64(blockData[i:i+8], op.Target)
}
}
// Store this block.
serialization, err := dvid.SerializeData(blockData, d.Compression(), d.Checksum())
if err != nil {
dvid.Criticalf("Unable to serialize block in %q: %v\n", d.DataName(), err)
return
}
if err := store.Put(ctx, tk, serialization); err != nil {
dvid.Errorf("Error in putting key %v: %v\n", tk, err)
}
// Notify any downstream downres instance.
d.publishBlockChange(ctx.VersionID(), op.mutID, op.block, blockData)
}
func (d *Data) deleteElementInLabel(ctx *datastore.VersionedCtx, batch storage.Batch, pt dvid.Point3d) error {
labelData := d.GetSyncedLabelblk()
if labelData == nil {
return nil // no synced labels
}
label, err := labelData.GetLabelAtPoint(ctx.VersionID(), pt)
if err != nil {
return err
}
tk := NewLabelTKey(label)
elems, err := getElementsNR(ctx, tk)
if err != nil {
return fmt.Errorf("err getting elements for label %d: %v\n", label, err)
}
// Note all elements to be deleted.
var delta DeltaModifyElements
var toDel []int
for i, elem := range elems {
if pt.Equals(elem.Pos) {
delta.Del = append(delta.Del, ElementPos{Label: label, Kind: elem.Kind, Pos: elem.Pos})
toDel = append(toDel, i)
}
}
if len(toDel) == 0 {
return nil
}
// Delete them from high index to low index due while reusing slice.
for i := len(toDel) - 1; i >= 0; i-- {
d := toDel[i]
elems[d] = elems[len(elems)-1]
elems[len(elems)-1] = ElementNR{}
elems = elems[:len(elems)-1]
}
// Put the modified list of elements
if err := putBatchElements(batch, tk, elems); err != nil {
return err
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
return nil
}
func (d *Data) moveElementInLabels(ctx *datastore.VersionedCtx, batch storage.Batch, from, to dvid.Point3d, moved ElementNR) error {
labelData := d.GetSyncedLabelblk()
if labelData == nil {
return nil // no label denormalization possible
}
oldLabel, err := labelData.GetLabelAtPoint(ctx.VersionID(), from)
if err != nil {
return err
}
newLabel, err := labelData.GetLabelAtPoint(ctx.VersionID(), to)
if err != nil {
return err
}
if oldLabel == newLabel {
return nil
}
var delta DeltaModifyElements
if oldLabel != 0 {
tk := NewLabelTKey(oldLabel)
elems, err := getElementsNR(ctx, tk)
if err != nil {
return fmt.Errorf("err getting elements for label %d: %v", oldLabel, err)
}
if _, changed := elems.delete(from); changed {
if err := putBatchElements(batch, tk, elems); err != nil {
return fmt.Errorf("err putting deleted label %d element: %v", oldLabel, err)
}
delta.Del = append(delta.Del, ElementPos{Label: oldLabel, Kind: moved.Kind, Pos: from})
}
}
if newLabel != 0 {
tk := NewLabelTKey(newLabel)
elems, err := getElementsNR(ctx, tk)
if err != nil {
return fmt.Errorf("err getting elements for label %d: %v", newLabel, err)
}
elems.add(ElementsNR{moved})
if err := putBatchElements(batch, tk, elems); err != nil {
return err
}
delta.Add = append(delta.Add, ElementPos{Label: newLabel, Kind: moved.Kind, Pos: to})
}
// Notify any subscribers of label annotation changes.
if len(delta.Del) != 0 || len(delta.Add) != 0 {
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
}
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
default:
}
// Get the data name and parse out the channel number or see if composite is required.
var channelNum int32
channumStr := strings.TrimPrefix(parts[2], string(d.DataName()))
if len(channumStr) == 0 {
channelNum = 0
} else {
n, err := strconv.ParseInt(channumStr, 10, 32)
if err != nil {
server.BadRequest(w, r, "Error parsing channel number from data name '%s': %v", parts[2], err)
return
}
if int(n) > d.NumChannels {
minChannelName := fmt.Sprintf("%s1", d.DataName())
maxChannelName := fmt.Sprintf("%s%d", d.DataName(), d.NumChannels)
server.BadRequest(w, r, "Data only has %d channels. Use names '%s' -> '%s'", d.NumChannels,
minChannelName, maxChannelName)
return
}
channelNum = int32(n)
}
// Get the data shape.
shapeStr := dvid.DataShapeString(parts[3])
dataShape, err := shapeStr.DataShape()
if err != nil {
server.BadRequest(w, r, "Bad data shape given '%s'", shapeStr)
return
}
switch dataShape.ShapeDimensions() {
case 2:
sizeStr, offsetStr := parts[4], parts[5]
slice, err := dvid.NewSliceFromStrings(shapeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "post" {
server.BadRequest(w, r, "DVID does not yet support POST of slices into multichannel data")
return
} else {
if d.NumChannels == 0 || d.Data.Values == nil {
server.BadRequest(w, r, "Cannot retrieve absent data '%d'. Please load data.", d.DataName())
return
}
values := d.Data.Values
if len(values) <= int(channelNum) {
server.BadRequest(w, r, "Must choose channel from 0 to %d", len(values))
return
}
stride := slice.Size().Value(0) * values.BytesPerElement()
dataValues := dvid.DataValues{values[channelNum]}
data := make([]uint8, int(slice.NumVoxels()))
v := imageblk.NewVoxels(slice, dataValues, data, stride)
channel := &Channel{
Voxels: v,
channelNum: channelNum,
}
img, err := d.GetImage(ctx.VersionID(), channel.Voxels, nil)
var formatStr string
if len(parts) >= 7 {
formatStr = parts[6]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
case 3:
sizeStr, offsetStr := parts[4], parts[5]
_, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
server.BadRequest(w, r, "DVID does not yet support GET of volume data")
return
} else {
server.BadRequest(w, r, "DVID does not yet support POST of volume data")
return
}
default:
server.BadRequest(w, r, "DVID does not yet support nD volumes")
return
}
timedLog.Infof("HTTP %s: %s", r.Method, dataShape)
}
// Note that this does not delete any removed labels in the block since we only get the CURRENT block
// and not PAST blocks. To allow mutation of label blocks, not just ingestion, we need another function.
func (d *Data) ingestBlock(ctx *datastore.VersionedCtx, block imageblk.Block, batcher storage.KeyValueBatcher) {
// Iterate through this block of labels and create RLEs for each label.
blockBytes := len(block.Data)
if blockBytes != int(d.BlockSize.Prod())*8 {
dvid.Criticalf("Deserialized label block %d bytes, not uint64 size times %d block elements\n",
blockBytes, d.BlockSize.Prod())
return
}
labelRLEs := make(map[uint64]dvid.RLEs, 10)
firstPt := block.Index.MinPoint(d.BlockSize)
lastPt := block.Index.MaxPoint(d.BlockSize)
var curStart dvid.Point3d
var voxelLabel, curLabel, maxLabel uint64
var z, y, x, curRun int32
start := 0
for z = firstPt.Value(2); z <= lastPt.Value(2); z++ {
for y = firstPt.Value(1); y <= lastPt.Value(1); y++ {
for x = firstPt.Value(0); x <= lastPt.Value(0); x++ {
voxelLabel = binary.LittleEndian.Uint64(block.Data[start : start+8])
if maxLabel < voxelLabel {
maxLabel = voxelLabel
}
start += 8
// If we hit background or have switched label, save old run and start new one.
if voxelLabel == 0 || voxelLabel != curLabel {
// Save old run
if curRun > 0 {
labelRLEs[curLabel] = append(labelRLEs[curLabel], dvid.NewRLE(curStart, curRun))
}
// Start new one if not zero label.
if voxelLabel != 0 {
curStart = dvid.Point3d{x, y, z}
curRun = 1
} else {
curRun = 0
}
curLabel = voxelLabel
} else {
curRun++
}
}
// Force break of any runs when we finish x scan.
if curRun > 0 {
labelRLEs[curLabel] = append(labelRLEs[curLabel], dvid.NewRLE(curStart, curRun))
curLabel = 0
curRun = 0
}
}
}
// Store the RLEs for each label in this block.
if maxLabel > 0 {
batch := batcher.NewBatch(ctx)
blockStr := block.Index.ToIZYXString()
for label, rles := range labelRLEs {
tk := NewTKey(label, blockStr)
rleBytes, err := rles.MarshalBinary()
if err != nil {
dvid.Errorf("Bad encoding labelvol keys for label %d: %v\n", label, err)
return
}
batch.Put(tk, rleBytes)
}
// compare-and-set MaxLabel and batch commit
d.casMaxLabel(batch, ctx.VersionID(), maxLabel)
}
}
// If a block of labels is deleted, the associated synapse elements should be changed to zero label elements.
func (d *Data) deleteBlock(ctx *datastore.VersionedCtx, block labels.DeleteBlock, batcher storage.KeyValueBatcher) {
// Get the synaptic elements for this block
chunkPt := dvid.ChunkPoint3d(*block.Index)
tk := NewBlockTKey(chunkPt)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for block %s: %v\n", chunkPt, err)
return
}
if len(elems) == 0 {
return
}
blockSize := d.blockSize()
batch := batcher.NewBatch(ctx)
// Compute the strides (in bytes)
bX := blockSize[0] * 8
bY := blockSize[1] * bX
// Iterate through all element positions, finding corresponding label and storing elements.
toDel := LabelPoints{}
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := pt[2]*bY + pt[1]*bX + pt[0]*8
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
toDel.add(label, elem.Pos)
}
// Delete any non-zero label elements from their respective label k/v.
var delta DeltaModifyElements
for label, pts := range toDel {
tk := NewLabelTKey(label)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for label %d: %v\n", label, err)
return
}
save := false
for _, pt := range pts {
deleted, changed := elems.delete(pt)
if changed {
save = true
delta.Del = append(delta.Del, ElementPos{Label: label, Kind: deleted.Kind, Pos: pt})
}
}
if save {
if len(elems) == 0 {
batch.Delete(tk)
} else {
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
return
}
batch.Put(tk, val)
}
}
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in annotations %q after delete block: %v\n", d.DataName(), err)
return
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
}
// If a block of labels is mutated, adjust any label that was either removed or added.
func (d *Data) mutateBlock(ctx *datastore.VersionedCtx, block imageblk.MutatedBlock, batcher storage.KeyValueBatcher) {
// Get the synaptic elements for this block
chunkPt := dvid.ChunkPoint3d(*block.Index)
tk := NewBlockTKey(chunkPt)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for block %s: %v\n", chunkPt, err)
return
}
if len(elems) == 0 {
return
}
blockSize := d.blockSize()
batch := batcher.NewBatch(ctx)
// Compute the strides (in bytes)
bX := blockSize[0] * 8
bY := blockSize[1] * bX
// Iterate through all element positions, finding corresponding label and storing elements.
var delta DeltaModifyElements
labels := make(map[uint64]struct{})
toAdd := LabelElements{}
toDel := LabelPoints{}
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := pt[2]*bY + pt[1]*bX + pt[0]*8
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
var prev uint64
if len(block.Prev) != 0 {
prev = binary.LittleEndian.Uint64(block.Prev[i : i+8])
}
if label == prev {
continue
}
if label != 0 {
toAdd.add(label, elem)
labels[label] = struct{}{}
delta.Add = append(delta.Add, ElementPos{Label: label, Kind: elem.Kind, Pos: elem.Pos})
}
if prev != 0 {
toDel.add(prev, elem.Pos)
labels[prev] = struct{}{}
delta.Del = append(delta.Del, ElementPos{Label: prev, Kind: elem.Kind, Pos: elem.Pos})
}
}
// Modify any modified label k/v.
for label := range labels {
tk := NewLabelTKey(label)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for label %d: %v\n", label, err)
return
}
additions, found := toAdd[label]
if found {
elems.add(additions)
}
deletions, found := toDel[label]
if found {
for _, pt := range deletions {
elems.delete(pt)
}
}
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
return
}
batch.Put(tk, val)
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in annotations %q after delete block: %v\n", d.DataName(), err)
return
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
}
func (d *Data) mutateBlock(ctx *datastore.VersionedCtx, block imageblk.MutatedBlock, batcher storage.KeyValueBatcher) {
// Iterate through previous and current labels, detecting set of previous labels and RLEs for current labels.
blockBytes := len(block.Data)
if blockBytes != int(d.BlockSize.Prod())*8 {
dvid.Criticalf("Deserialized label block %d bytes, not uint64 size times %d block elements\n",
blockBytes, d.BlockSize.Prod())
return
}
labelRLEs := make(map[uint64]dvid.RLEs, 10)
labelDiff := make(map[uint64]bool, 10)
firstPt := block.Index.MinPoint(d.BlockSize)
lastPt := block.Index.MaxPoint(d.BlockSize)
var curStart dvid.Point3d
var voxelLabel, curLabel, maxLabel uint64
var z, y, x, curRun int32
start := 0
for z = firstPt.Value(2); z <= lastPt.Value(2); z++ {
for y = firstPt.Value(1); y <= lastPt.Value(1); y++ {
for x = firstPt.Value(0); x <= lastPt.Value(0); x++ {
var pastLabel uint64
if block.Prev == nil || len(block.Prev) == 0 {
pastLabel = 0
} else {
pastLabel = binary.LittleEndian.Uint64(block.Prev[start : start+8])
}
voxelLabel = binary.LittleEndian.Uint64(block.Data[start : start+8])
if maxLabel < voxelLabel {
maxLabel = voxelLabel
}
if pastLabel != 0 {
if pastLabel != voxelLabel {
labelDiff[pastLabel] = true
} else {
_, found := labelDiff[pastLabel]
if !found {
labelDiff[pastLabel] = false
}
}
}
start += 8
// If we hit background or have switched label, save old run and start new one.
if voxelLabel == 0 || voxelLabel != curLabel {
// Save old run
if curRun > 0 {
labelRLEs[curLabel] = append(labelRLEs[curLabel], dvid.NewRLE(curStart, curRun))
}
// Start new one if not zero label.
if voxelLabel != 0 {
curStart = dvid.Point3d{x, y, z}
curRun = 1
} else {
curRun = 0
}
curLabel = voxelLabel
} else {
curRun++
}
}
// Force break of any runs when we finish x scan.
if curRun > 0 {
labelRLEs[curLabel] = append(labelRLEs[curLabel], dvid.NewRLE(curStart, curRun))
curLabel = 0
curRun = 0
}
}
}
// If a previous label has no change with current label RLE, then delete the label RLE since no changes
// are necessary. Else if previous label is not present in current label RLEs, delete labelvol.
var deletes []storage.TKey
blockStr := block.Index.ToIZYXString()
for label, diff := range labelDiff {
_, found := labelRLEs[label]
if diff && !found {
// mark previous label's RLEs for deletion
tk := NewTKey(label, blockStr)
deletes = append(deletes, tk)
} else if !diff && found {
// delete current label's RLEs because there's no difference with past RLE
delete(labelRLEs, label)
}
}
if len(deletes) > 0 {
batch := batcher.NewBatch(ctx)
for _, tk := range deletes {
batch.Delete(tk)
}
if err := batch.Commit(); err != nil {
dvid.Errorf("batch commit on deleting previous labels' labelvols: %v\n", err)
}
}
// Store the RLEs for each label in this block that are new or modified.
if len(labelRLEs) > 0 {
batch := batcher.NewBatch(ctx)
for label, rles := range labelRLEs {
tk := NewTKey(label, blockStr)
rleBytes, err := rles.MarshalBinary()
if err != nil {
dvid.Errorf("Bad encoding labelvol keys for label %d: %v\n", label, err)
return
}
batch.Put(tk, rleBytes)
}
// compare-and-set MaxLabel and batch commit
d.casMaxLabel(batch, ctx.VersionID(), maxLabel)
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
// TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints.
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
case "delete":
default:
server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *imageblk.ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(imageblk.ROI)
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "post" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "label":
// GET <api URL>/node/<UUID>/<data name>/label/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)
case "labels":
// POST <api URL>/node/<UUID>/<data name>/labels
if action != "post" {
server.BadRequest(w, r, "Batch labels query must be a POST request")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.")
return
}
var coords []dvid.Point3d
if err := json.Unmarshal(data, &coords); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "[")
sep := false
for _, coord := range coords {
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
if sep {
fmt.Fprintf(w, ",")
}
fmt.Fprintf(w, "%d", label)
sep = true
}
fmt.Fprintf(w, "]")
timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))
case "blocks":
// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "delete" {
server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
return
}
if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)
case "pseudocolor":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
lbl, err := d.NewLabels(slice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
// Convert to pseudocolor
pseudoColor, err := colorImage(img)
if err != nil {
server.BadRequest(w, r, err)
return
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
return
}
case "raw", "isotropic":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
lbl, err := d.NewLabels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
compression := queryValues.Get("compression")
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
lbl, err := d.NewLabels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
switch compression {
case "":
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "lz4":
compressed := make([]byte, lz4.CompressBound(data))
var n, outSize int
if outSize, err = lz4.Compress(data, compressed); err != nil {
server.BadRequest(w, r, err)
return
}
compressed = compressed[:outSize]
if n, err = w.Write(compressed); err != nil {
server.BadRequest(w, r, err)
return
}
if n != outSize {
errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
dvid.Errorf(errmsg)
server.BadRequest(w, r, errmsg)
return
}
case "gzip":
gw := gzip.NewWriter(w)
if _, err = gw.Write(data); err != nil {
server.BadRequest(w, r, err)
return
}
if err = gw.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
} else {
if isotropic {
server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
var data []byte
switch compression {
case "":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read 3d uncompressed POST")
case "lz4":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
if len(data) == 0 {
server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
return
}
tlog.Debugf("read 3d lz4 POST")
tlog = dvid.NewTimeLog()
uncompressed := make([]byte, subvol.NumVoxels()*8)
err = lz4.Uncompress(data, uncompressed)
if err != nil {
server.BadRequest(w, r, err)
return
}
data = uncompressed
tlog.Debugf("uncompressed 3d lz4 POST")
case "gzip":
tlog := dvid.NewTimeLog()
gr, err := gzip.NewReader(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err = ioutil.ReadAll(gr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err = gr.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read and uncompress 3d gzip POST")
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
lbl, err := d.NewLabels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
return
}
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.",
parts[3], d.DataName())
}
}
func (d *Data) DeleteBlocks(ctx *datastore.VersionedCtx, start dvid.ChunkPoint3d, span int) error {
store, err := storage.MutableStore()
if err != nil {
return fmt.Errorf("Data type labelblk had error initializing store: %v\n", err)
}
batcher, ok := store.(storage.KeyValueBatcher)
if !ok {
return fmt.Errorf("Data type labelblk requires batch-enabled store, which %q is not\n", store)
}
indexBeg := dvid.IndexZYX(start)
end := start
end[0] += int32(span - 1)
indexEnd := dvid.IndexZYX(end)
begTKey := NewTKey(&indexBeg)
endTKey := NewTKey(&indexEnd)
iv := dvid.InstanceVersion{d.DataName(), ctx.VersionID()}
mapping := labels.MergeCache.LabelMap(iv)
kvs, err := store.GetRange(ctx, begTKey, endTKey)
if err != nil {
return err
}
batch := batcher.NewBatch(ctx)
uncompress := true
for _, kv := range kvs {
izyx, err := DecodeTKey(kv.K)
if err != nil {
return err
}
// Delete the labelblk (really tombstones it)
batch.Delete(kv.K)
// Send data to delete associated labelvol for labels in this block
block, _, err := dvid.DeserializeData(kv.V, uncompress)
if err != nil {
return fmt.Errorf("Unable to deserialize block, %s (%v): %v", ctx, kv.K, err)
}
if mapping != nil {
n := len(block) / 8
for i := 0; i < n; i++ {
orig := binary.LittleEndian.Uint64(block[i*8 : i*8+8])
mapped, found := mapping.FinalLabel(orig)
if !found {
mapped = orig
}
binary.LittleEndian.PutUint64(block[i*8:i*8+8], mapped)
}
}
// Notify any subscribers that we've deleted this block.
evt := datastore.SyncEvent{d.DataName(), labels.DeleteBlockEvent}
msg := datastore.SyncMessage{ctx.VersionID(), labels.DeleteBlock{izyx, block}}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
}
return batch.Commit()
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Check the action
action := strings.ToLower(r.Method)
if action != "get" {
server.BadRequest(w, r, "labelsz data can only accept GET HTTP requests")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "size":
// GET <api URL>/node/<UUID>/<data name>/size/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'size' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
size, err := d.GetSize(ctx.VersionID(), label)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "{%q: %d, %q: %d}", "Label", label, "Voxels", size)
timedLog.Infof("HTTP %s: get label %d size", r.Method, label)
case "sizerange":
// GET <api URL>/node/<UUID>/<data name>/sizerange/<min size>/<optional max size>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires at least the minimum size to follow 'sizerange' command")
return
}
minSize, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
var maxSize uint64
if len(parts) >= 6 {
maxSize, err = strconv.ParseUint(parts[5], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
jsonStr, err := d.GetSizeRange(ctx.VersionID(), minSize, maxSize)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: get labels with volume > %d and < %d (%s)", r.Method, minSize, maxSize, r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call '%s' for labelsz data '%s'. See API help.",
parts[3], d.DataName())
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "incomplete API specification")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
default:
}
// Get the key and process request
var comment string
command := parts[3]
method := strings.ToLower(r.Method)
switch command {
case "roi":
switch method {
case "get":
if !d.Ready {
w.WriteHeader(http.StatusPartialContent)
}
jsonBytes, err := Get(ctx)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP GET ROI %q: %d bytes\n", d.DataName(), len(jsonBytes))
case "post":
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
err = d.PutJSON(ctx.VersionID(), data)
if err != nil {
server.BadRequest(w, r, err)
return
}
comment = fmt.Sprintf("HTTP POST ROI %q: %d bytes\n", d.DataName(), len(data))
case "delete":
if err := d.Delete(ctx); err != nil {
server.BadRequest(w, r, err)
return
}
comment = fmt.Sprintf("HTTP DELETE ROI %q\n", d.DataName())
}
case "mask":
if method != "get" {
server.BadRequest(w, r, "ROI mask only supports GET")
return
}
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", command)
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 3:
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err := d.GetMask(ctx, subvol)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "Currently only 3d masks ('0_1_2' shape) is supported")
return
}
case "ptquery":
switch method {
case "get":
server.BadRequest(w, r, "ptquery requires POST with list of points")
return
case "post":
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
jsonBytes, err := d.PointQuery(ctx, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP POST ptquery '%s'\n", d.DataName())
}
case "partition":
if method != "get" {
server.BadRequest(w, r, "partition only supports GET request")
return
}
queryValues := r.URL.Query()
batchsizeStr := queryValues.Get("batchsize")
batchsize, err := strconv.Atoi(batchsizeStr)
if err != nil {
server.BadRequest(w, r, fmt.Sprintf("Error reading batchsize query string: %v", err))
return
}
var jsonBytes []byte
optimizedStr := queryValues.Get("optimized")
dvid.Infof("queryvalues = %v\n", queryValues)
if optimizedStr == "true" || optimizedStr == "on" {
dvid.Infof("Perform optimized partitioning into subvolumes using batchsize %d\n", batchsize)
jsonBytes, err = d.Partition(ctx, int32(batchsize))
} else {
dvid.Infof("Performing simple partitioning into subvolumes using batchsize %d\n", batchsize)
jsonBytes, err = d.SimplePartition(ctx, int32(batchsize))
}
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP partition '%s' with batch size %d\n",
d.DataName(), batchsize)
default:
w.Header().Set("Content-Type", "text/plain")
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
timedLog.Infof(comment)
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(ROI)
attenuationStr := queryValues.Get("attenuation")
if len(attenuationStr) != 0 {
attenuation, err := strconv.Atoi(attenuationStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if attenuation < 1 || attenuation > 7 {
server.BadRequest(w, r, "Attenuation should be from 1 to 7 (divides by 2^n)")
return
}
roiptr.attenuation = uint8(attenuation)
}
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
fmt.Printf("Setting configuration of data '%s'\n", d.DataName())
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
case "rawkey":
// GET <api URL>/node/<UUID>/<data name>/rawkey?x=<block x>&y=<block y>&z=<block z>
if len(parts) != 4 {
server.BadRequest(w, r, "rawkey endpoint should be followed by query strings (x, y, and z) giving block coord")
return
}
case "blocks":
// GET <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
// POST <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "%q must be followed by block-coord/span-x", parts[3])
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
data, err := d.GetBlocks(ctx.VersionID(), blockCoord, int32(span))
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
} else {
if err := d.PutBlocks(ctx.VersionID(), blockCoord, span, r.Body); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: Blocks (%s)", r.Method, r.URL)
case "arb":
// GET <api URL>/node/<UUID>/<data name>/arb/<top left>/<top right>/<bottom left>/<res>[/<format>]
if len(parts) < 8 {
server.BadRequest(w, r, "%q must be followed by top-left/top-right/bottom-left/res", parts[3])
return
}
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
img, err := d.GetArbitraryImage(ctx, parts[4], parts[5], parts[6], parts[7])
if err != nil {
server.BadRequest(w, r, err)
return
}
var formatStr string
if len(parts) >= 9 {
formatStr = parts[8]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: Arbitrary image (%s)", r.Method, r.URL)
case "raw", "isotropic":
// GET <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>]
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
if err != nil {
server.BadRequest(w, r, err)
return
}
vox, err := d.NewVoxels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), vox, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
if queryStrings.Get("throttle") == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
vox, err := d.NewVoxels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), vox, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
} else {
if isotropic {
err := fmt.Errorf("can only PUT 'raw' not 'isotropic' images")
server.BadRequest(w, r, err)
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store voxels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
vox, err := d.NewVoxels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), vox, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
}
default:
server.BadRequest(w, r, "Unrecognized API call for voxels %q. See API help.", d.DataName())
}
}
// stores synaptic elements arranged by label, replacing any
// elements at same position.
func (d *Data) storeLabelElements(ctx *datastore.VersionedCtx, batch storage.Batch, be blockElements) error {
labelData := d.GetSyncedLabelblk()
if labelData == nil {
dvid.Infof("No synced labels for annotation %q, skipping label-aware denormalization.\n", d.DataName())
return nil // no synced labels
}
// Compute the strides (in bytes)
blockSize := d.blockSize()
bX := blockSize[0] * 8
bY := blockSize[1] * bX
blockBytes := int(blockSize[0] * blockSize[1] * blockSize[2] * 8)
toAdd := LabelElements{}
for izyxStr, elems := range be {
blockCoord, err := izyxStr.ToChunkPoint3d()
if err != nil {
return err
}
// Get the labels for this block
labels, err := labelData.GetLabelBlock(ctx.VersionID(), blockCoord)
if err != nil {
return err
}
if len(labels) == 0 {
continue
}
if len(labels) != blockBytes {
return fmt.Errorf("Expected %d bytes in %q label block, got %d instead. Aborting.", blockBytes, d.DataName(), len(labels))
}
// Group annotations by label
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := pt[2]*bY + pt[1]*bX + pt[0]*8
label := binary.LittleEndian.Uint64(labels[i : i+8])
if label != 0 {
toAdd.add(label, elem.ElementNR)
}
}
}
// Store all the added annotations to the appropriate labels.
var delta DeltaModifyElements
for label, additions := range toAdd {
tk := NewLabelTKey(label)
elems, err := getElementsNR(ctx, tk)
if err != nil {
return fmt.Errorf("err getting elements for label %d: %v\n", label, err)
}
// Check if these annotations already exist.
emap := make(map[string]int)
for i, elem := range elems {
emap[elem.Pos.MapKey()] = i
}
for _, elem := range additions {
i, found := emap[elem.Pos.MapKey()]
if !found {
elems = append(elems, elem)
delta.Add = append(delta.Add, ElementPos{Label: label, Kind: elem.Kind, Pos: elem.Pos})
} else {
elems[i] = elem // replace properties if same position
}
}
if err := putBatchElements(batch, tk, elems); err != nil {
return fmt.Errorf("couldn't serialize label %d annotations in instance %q: %v\n", label, d.DataName(), err)
}
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
return err
}
return nil
}
// If a block of labels is ingested, adjust each label's synaptic element list.
func (d *Data) ingestBlock(ctx *datastore.VersionedCtx, block imageblk.Block, batcher storage.KeyValueBatcher) {
// Get the synaptic elements for this block
chunkPt := dvid.ChunkPoint3d(*block.Index)
tk := NewBlockTKey(chunkPt)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for block %s: %v\n", chunkPt, err)
return
}
if len(elems) == 0 {
return
}
blockSize := d.blockSize()
batch := batcher.NewBatch(ctx)
// Compute the strides (in bytes)
bX := blockSize[0] * 8
bY := blockSize[1] * bX
// Iterate through all element positions, finding corresponding label and storing elements.
added := 0
toAdd := LabelElements{}
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := (pt[2]*bY+pt[1])*bX + pt[0]*8
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
if label != 0 {
toAdd.add(label, elem)
added++
}
}
// Add any non-zero label elements to their respective label k/v.
var delta DeltaModifyElements
delta.Add = make([]ElementPos, added)
i := 0
for label, addElems := range toAdd {
tk := NewLabelTKey(label)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for label %d: %v\n", label, err)
return
}
elems.add(addElems)
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
return
}
batch.Put(tk, val)
for _, addElem := range addElems {
delta.Add[i] = ElementPos{Label: label, Kind: addElem.Kind, Pos: addElem.Pos}
i++
}
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in annotations %q after delete block: %v\n", d.DataName(), err)
return
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Check the action
action := strings.ToLower(r.Method)
if action != "get" {
server.BadRequest(w, r, "labelsurf data can only accept GET HTTP requests")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "surface":
// GET <api URL>/node/<UUID>/<data name>/surface/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'surface' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
fmt.Printf("Getting surface for label %d\n", label)
if err != nil {
server.BadRequest(w, r, err)
return
}
gzipData, found, err := GetSurface(ctx, label)
if err != nil {
server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
return
}
w.Header().Set("Content-type", "application/octet-stream")
if err := dvid.WriteGzip(gzipData, w, r); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: surface on label %d (%s)", r.Method, label, r.URL)
case "surface-by-point":
// GET <api URL>/node/<UUID>/<data name>/surface-by-point/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'surface-by-point' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
gzipData, found, err := GetSurface(ctx, label)
if err != nil {
server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
return
}
fmt.Printf("Found surface for label %d: %d bytes (gzip payload)\n", label, len(gzipData))
w.Header().Set("Content-type", "application/octet-stream")
if err := dvid.WriteGzip(gzipData, w, r); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: surface-by-point at %s (%s)", r.Method, coord, r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call '%s' for labelsurf data '%s'. See API help.",
parts[3], d.DataName())
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
versionID := ctx.VersionID()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "sparsevol":
// GET <api URL>/node/<UUID>/<data name>/sparsevol/<label>
// POST <api URL>/node/<UUID>/<data name>/sparsevol/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
switch action {
case "get":
queryValues := r.URL.Query()
var b Bounds
b.VoxelBounds, err = dvid.BoundsFromQueryString(r)
if err != nil {
server.BadRequest(w, r, "Error parsing bounds from query string: %v\n", err)
return
}
b.BlockBounds = b.VoxelBounds.Divide(d.BlockSize)
b.Exact = queryValues.Get("exact") == "true"
data, err := GetSparseVol(ctx, label, b)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "post":
server.BadRequest(w, r, "POST of sparsevol not currently implemented\n")
return
// if err := d.PutSparseVol(versionID, label, r.Body); err != nil {
// server.BadRequest(w, r, err)
// return
// }
default:
server.BadRequest(w, r, "Unable to handle HTTP action %s on sparsevol endpoint", action)
return
}
timedLog.Infof("HTTP %s: sparsevol on label %d (%s)", r.Method, label, r.URL)
case "sparsevol-by-point":
// GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'sparsevol-by-point' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
source, err := d.GetSyncedLabelblk(versionID)
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := source.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err := GetSparseVol(ctx, label, Bounds{})
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: sparsevol-by-point at %s (%s)", r.Method, coord, r.URL)
case "sparsevol-coarse":
// GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol-coarse' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err := GetSparseCoarseVol(ctx, label)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: sparsevol-coarse on label %d (%s)", r.Method, label, r.URL)
case "maxlabel":
// GET <api URL>/node/<UUID>/<data name>/maxlabel
w.Header().Set("Content-Type", "application/json")
switch action {
case "get":
maxlabel, ok := d.MaxLabel[versionID]
if !ok {
server.BadRequest(w, r, "No maximum label found for %s version %d\n", d.DataName(), versionID)
return
}
fmt.Fprintf(w, "{%q: %d}", "maxlabel", maxlabel)
default:
server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
return
}
timedLog.Infof("HTTP maxlabel request (%s)", r.URL)
case "nextlabel":
// GET <api URL>/node/<UUID>/<data name>/nextlabel
// POST <api URL>/node/<UUID>/<data name>/nextlabel
w.Header().Set("Content-Type", "application/json")
switch action {
case "get":
fmt.Fprintf(w, "{%q: %d}", "nextlabel", d.MaxRepoLabel+1)
case "post":
server.BadRequest(w, r, "POST on maxlabel is not supported yet.\n")
return
default:
server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
return
}
timedLog.Infof("HTTP maxlabel request (%s)", r.URL)
case "split":
// POST <api URL>/node/<UUID>/<data name>/split/<label>
if action != "post" {
server.BadRequest(w, r, "Split requests must be POST actions.")
return
}
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command")
return
}
fromLabel, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
toLabel, err := d.SplitLabels(ctx.VersionID(), fromLabel, r.Body)
if err != nil {
server.BadRequest(w, r, fmt.Sprintf("Error on split: %v", err))
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "{%q: %d}", "label", toLabel)
timedLog.Infof("HTTP split request (%s)", r.URL)
case "merge":
// POST <api URL>/node/<UUID>/<data name>/merge
if action != "post" {
server.BadRequest(w, r, "Merge requests must be POST actions.")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for merge. Should be JSON.")
return
}
var tuple labels.MergeTuple
if err := json.Unmarshal(data, &tuple); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad merge op JSON: %v", err))
return
}
mergeOp, err := tuple.Op()
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.MergeLabels(ctx.VersionID(), mergeOp); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Error on merge: %v", err))
return
}
timedLog.Infof("HTTP merge request (%s)", r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelvol data %q. See API help.",
parts[3], d.DataName())
}
}