// call KVAutobus key_range API
func (db *KVAutobus) getKeyRange(kStart, kEnd storage.Key) (Ks, error) {
b64key1 := encodeKey(kStart)
b64key2 := encodeKey(kEnd)
url := fmt.Sprintf("%s/kvautobus/api/key_range/%s/%s/%s/", db.host, db.collection, b64key1, b64key2)
timedLog := dvid.NewTimeLog()
resp, err := db.client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil // Handle no keys found.
}
r := msgp.NewReader(bufio.NewReader(resp.Body))
var mks Ks
if err := mks.DecodeMsg(r); err != nil {
return nil, err
}
for _, mk := range mks {
storage.StoreKeyBytesRead <- len(mk)
}
timedLog.Infof("PROXY key_range to %s returned %d (%d keys)\n", db.host, resp.StatusCode, len(mks))
return mks, nil
}
// check if any metadata has been written into this store.
func (db *KVAutobus) metadataExists() (bool, error) {
var ctx storage.MetadataContext
kStart, kEnd := ctx.KeyRange()
b64key1 := encodeKey(kStart)
b64key2 := encodeKey(kEnd)
url := fmt.Sprintf("%s/kvautobus/api/key_range/%s/%s/%s/", db.host, db.collection, b64key1, b64key2)
dvid.Infof("metdataExists: doing GET on %s\n", url)
timedLog := dvid.NewTimeLog()
resp, err := db.client.Get(url)
if err != nil {
return false, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return true, nil // Handle no keys found.
}
r := msgp.NewReader(bufio.NewReader(resp.Body))
var mks Ks
if err := mks.DecodeMsg(r); err != nil {
return false, err
}
timedLog.Infof("PROXY key_range metadata to %s returned %d (%d keys)\n", db.host, resp.StatusCode, len(mks))
if len(mks) == 0 {
return false, nil
}
return true, nil
}
// call KVAutobus keyvalue_range API
func (db *KVAutobus) getKVRange(kStart, kEnd storage.Key) (KVs, error) {
b64key1 := encodeKey(kStart)
b64key2 := encodeKey(kEnd)
url := fmt.Sprintf("%s/kvautobus/api/keyvalue_range/%s/%s/", db.host, b64key1, b64key2)
timedLog := dvid.NewTimeLog()
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil // Handle no keys found.
}
r := msgp.NewReader(bufio.NewReader(resp.Body))
var mkvs KVs
if err := mkvs.DecodeMsg(r); err != nil {
dvid.Errorf("Couldn't decode getKVRange return\n")
return nil, err
}
timedLog.Infof("PROXY keyvalue_range to %s returned %d (%d kv pairs)\n", db.host, resp.StatusCode, len(mkvs))
return mkvs, nil
}
// GetVoxels copies voxels from the storage engine to Voxels, a requested subvolume or 2d image.
func (d *Data) GetVoxels(v dvid.VersionID, vox *Voxels, r *ROI) error {
timedLog := dvid.NewTimeLog()
defer timedLog.Infof("GetVoxels %s", vox)
store, err := storage.MutableStore()
if err != nil {
return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
// Only do one request at a time, although each request can start many goroutines.
server.SpawnGoroutineMutex.Lock()
defer server.SpawnGoroutineMutex.Unlock()
ctx := datastore.NewVersionedCtx(d, v)
wg := new(sync.WaitGroup)
for it, err := vox.IndexIterator(d.BlockSize()); err == nil && it.Valid(); it.NextSpan() {
indexBeg, indexEnd, err := it.IndexSpan()
if err != nil {
return err
}
begTKey := NewTKey(indexBeg)
endTKey := NewTKey(indexEnd)
// Get set of blocks in ROI if ROI provided
var chunkOp *storage.ChunkOp
if r != nil && r.Iter != nil {
ptBeg := indexBeg.Duplicate().(dvid.ChunkIndexer)
ptEnd := indexEnd.Duplicate().(dvid.ChunkIndexer)
begX := ptBeg.Value(0)
endX := ptEnd.Value(0)
blocksInROI := make(map[string]bool, (endX - begX + 1))
c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)}
for x := begX; x <= endX; x++ {
c[0] = x
curIndex := dvid.IndexZYX(c)
if r.Iter.InsideFast(curIndex) {
indexString := string(curIndex.Bytes())
blocksInROI[indexString] = true
}
}
chunkOp = &storage.ChunkOp{&getOperation{vox, blocksInROI, r.attenuation}, wg}
} else {
chunkOp = &storage.ChunkOp{&getOperation{vox, nil, 0}, wg}
}
// Send the entire range of key-value pairs to chunk processor
err = store.ProcessRange(ctx, begTKey, endTKey, chunkOp, storage.ChunkFunc(d.ReadChunk))
if err != nil {
return fmt.Errorf("Unable to GET data %s: %v", ctx, err)
}
}
if err != nil {
return err
}
wg.Wait()
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
action := strings.ToLower(r.Method)
switch action {
case "get":
// Acceptable
default:
server.BadRequest(w, r, "googlevoxels can only handle GET HTTP verbs at this time")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "incomplete API request")
return
}
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "tile":
if err := d.ServeTile(w, r, parts); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL)
case "raw":
if err := d.ServeImage(w, r, parts); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: image (%s)", r.Method, r.URL)
default:
server.BadRequest(w, r, "Illegal request for googlevoxels data. See 'help' for REST API")
}
}
func (d *Data) processSplit(v dvid.VersionID, delta labels.DeltaSplit) {
timedLog := dvid.NewTimeLog()
mutID := d.NewMutationID()
splitOpStart := labels.DeltaSplitStart{delta.OldLabel, delta.NewLabel}
splitOpEnd := labels.DeltaSplitEnd{delta.OldLabel, delta.NewLabel}
iv := dvid.InstanceVersion{d.DataUUID(), v}
labels.SplitStart(iv, splitOpStart)
d.StartUpdate()
if delta.Split == nil {
// Coarse Split
for _, izyxStr := range delta.SortedBlocks {
n := izyxStr.Hash(numBlockHandlers)
d.MutAdd(mutID)
op := splitOp{
mutID: mutID,
oldLabel: delta.OldLabel,
newLabel: delta.NewLabel,
block: izyxStr,
}
d.procCh[n] <- procMsg{op: op, v: v}
}
} else {
// Fine Split
for izyxStr, blockRLEs := range delta.Split {
n := izyxStr.Hash(numBlockHandlers)
d.MutAdd(mutID)
op := splitOp{
mutID: mutID,
oldLabel: delta.OldLabel,
newLabel: delta.NewLabel,
rles: blockRLEs,
block: izyxStr,
}
d.procCh[n] <- procMsg{op: op, v: v}
}
}
// Wait for all blocks to be split then mark end of split op.
go func() {
d.MutWait(mutID)
d.MutDelete(mutID)
labels.SplitStop(iv, splitOpEnd)
timedLog.Debugf("labelblk sync complete for split of %d -> %d", delta.OldLabel, delta.NewLabel)
d.StopUpdate()
d.publishDownresCommit(v, mutID)
}()
}
func (db *KVAutobus) deleteRange(kStart, kEnd storage.Key) error {
b64key1 := encodeKey(kStart)
b64key2 := encodeKey(kEnd)
url := fmt.Sprintf("%s/kvautobus/api/keyvalue_range/%s/%s/%s/", db.host, db.collection, b64key1, b64key2)
timedLog := dvid.NewTimeLog()
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
defer resp.Body.Close()
resp, err := db.client.Do(req)
if err != nil {
return err
}
timedLog.Infof("PROXY delete keyvalue_range to %s returned %d\n", db.host, resp.StatusCode)
return nil
}
// LoadImages bulk loads images using different techniques if it is a multidimensional
// file like HDF5 or a sequence of PNG/JPG/TIF images.
func (d *Data) LoadImages(v dvid.VersionID, offset dvid.Point, filenames []string) error {
if len(filenames) == 0 {
return nil
}
timedLog := dvid.NewTimeLog()
// We only want one PUT on given version for given data to prevent interleaved
// chunk PUTs that could potentially overwrite slice modifications.
ctx := storage.NewDataContext(d, v)
loadMutex := ctx.Mutex()
loadMutex.Lock()
// Handle cleanup given multiple goroutines still writing data.
load := &bulkLoadInfo{filenames: filenames, versionID: v, offset: offset}
defer func() {
loadMutex.Unlock()
if load.extentChanged.Value() {
err := datastore.SaveDataByVersion(v, d)
if err != nil {
dvid.Errorf("Error in trying to save repo for voxel extent change: %v\n", err)
}
}
}()
// Use different loading techniques if we have a potentially multidimensional HDF5 file
// or many 2d images.
var err error
if dvid.Filename(filenames[0]).HasExtensionPrefix("hdf", "h5") {
err = d.loadHDF(load)
} else {
err = d.loadXYImages(load)
}
if err != nil {
timedLog.Infof("RPC load of %d files had error: %v\n", err)
} else {
timedLog.Infof("RPC load of %d files completed.\n", len(filenames))
}
return err
}
func (d *Data) processMerge(v dvid.VersionID, delta labels.DeltaMerge) {
timedLog := dvid.NewTimeLog()
d.StartUpdate()
mutID := d.NewMutationID()
for izyxStr := range delta.Blocks {
n := izyxStr.Hash(numBlockHandlers)
d.MutAdd(mutID)
op := mergeOp{mutID: mutID, MergeOp: delta.MergeOp, block: izyxStr}
d.procCh[n] <- procMsg{op: op, v: v}
}
// When we've processed all the delta blocks, we can remove this merge op
// from the merge cache since all labels will have completed.
go func() {
d.MutWait(mutID)
d.MutDelete(mutID)
timedLog.Debugf("labelblk sync complete for merge (%d blocks) of %s -> %d", len(delta.Blocks), delta.MergeOp.Merged, delta.MergeOp.Target)
d.StopUpdate()
d.publishDownresCommit(v, mutID)
}()
}
// call KVAutobus keyvalue_range API
func (db *KVAutobus) getKVRange(ctx storage.Context, kStart, kEnd storage.Key) (KVs, error) {
// Get any request context and pass to kvautobus for tracking.
reqctx, ok := ctx.(storage.RequestCtx)
var reqID string
if ok {
parts := strings.Split(reqctx.GetRequestID(), "/")
if len(parts) > 0 {
reqID = parts[len(parts)-1]
}
}
// Construct the KVAutobus URL
b64key1 := encodeKey(kStart)
b64key2 := encodeKey(kEnd)
url := fmt.Sprintf("%s/kvautobus/api/keyvalue_range/%s/%s/%s/?=%s", db.host, db.collection, b64key1, b64key2, reqID)
timedLog := dvid.NewTimeLog()
resp, err := db.client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil // Handle no keys found.
}
r := msgp.NewReader(bufio.NewReader(resp.Body))
var mkvs KVs
if err := mkvs.DecodeMsg(r); err != nil {
dvid.Errorf("Couldn't decode getKVRange return\n")
return nil, err
}
for _, mkv := range mkvs {
storage.StoreKeyBytesRead <- len(mkv[0])
storage.StoreValueBytesRead <- len(mkv[1])
}
timedLog.Infof("[%s] PROXY keyvalue_range to %s returned %d (%d kv pairs)\n", reqID, db.host, resp.StatusCode, len(mkvs))
return mkvs, nil
}
func (db *KVAutobus) RawGet(key storage.Key) ([]byte, error) {
b64key := encodeKey(key)
url := fmt.Sprintf("%s/kvautobus/api/value/%s/", db.host, b64key)
timedLog := dvid.NewTimeLog()
resp, err := http.Get(url)
if err != nil {
return nil, err
}
timedLog.Infof("PROXY get to %s returned %d\n", db.host, resp.StatusCode)
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil // Handle no key found.
}
r := msgp.NewReader(bufio.NewReader(resp.Body))
var bin Binary
if err := bin.DecodeMsg(r); err != nil {
return nil, err
}
return []byte(bin), nil
}
func (d *Data) serveVolume(w http.ResponseWriter, r *http.Request, geom *GoogleSubvolGeom, noblanks bool) error {
// If it's outside, write blank tile unless user wants no blanks.
if geom.outside {
if noblanks {
http.NotFound(w, r)
return fmt.Errorf("Requested subvolume is outside of available volume.")
}
return nil
}
// If we are within volume, get data from Google.
url, err := geom.GetURL(d.VolumeID, "")
if err != nil {
return err
}
timedLog := dvid.NewTimeLog()
client, err := d.GetClient()
if err != nil {
dvid.Errorf("Can't get OAuth2 connection to Google: %v\n", err)
return err
}
resp, err := client.Get(url)
if err != nil {
return err
}
timedLog.Infof("PROXY HTTP to Google: %s, returned response %d", url, resp.StatusCode)
defer resp.Body.Close()
// If it's on edge, we need to pad the subvolume to the requested size.
if geom.edge {
return fmt.Errorf("Googlevoxels subvolume GET does not pad data on edge at this time")
}
// If we aren't on edge or outside, our return status should be OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Unexpected status code %d on volume request (%q, volume id %q)", resp.StatusCode, d.DataName(), d.VolumeID)
}
w.Header().Set("Content-type", "application/octet-stream")
queryStrings := r.URL.Query()
compression := queryStrings.Get("compression")
switch compression {
case "lz4":
// Decompress snappy
sdata, err := ioutil.ReadAll(resp.Body)
timedLog.Infof("Got snappy-encoded subvolume from Google, %d bytes\n", len(sdata))
if err != nil {
return err
}
data, err := snappy.Decode(nil, sdata)
if err != nil {
return err
}
// Recompress and transmit as lz4
lz4data := make([]byte, lz4.CompressBound(data))
outSize, err := lz4.Compress(data, lz4data)
if err != nil {
return err
}
if _, err := w.Write(lz4data[:outSize]); err != nil {
return err
}
timedLog.Infof("Sent lz4-encoded subvolume from DVID, %d bytes\n", outSize)
default: // "snappy"
// Just stream data from Google
respBytes := 0
const BufferSize = 32 * 1024
buf := make([]byte, BufferSize)
for {
n, err := resp.Body.Read(buf)
respBytes += n
eof := (err == io.EOF)
if err != nil && !eof {
return err
}
if _, err = w.Write(buf[:n]); err != nil {
return err
}
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
if eof {
break
}
}
timedLog.Infof("Proxied snappy-encoded subvolume from Google, %d bytes\n", respBytes)
}
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
action := strings.ToLower(r.Method)
switch action {
case "get", "post":
// Acceptable
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "incomplete API request")
return
}
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "tile":
if action == "post" {
server.BadRequest(w, r, "DVID does not yet support POST of imagetile")
return
}
if err := d.ServeTile(uuid, ctx, w, r, parts); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL)
case "raw", "isotropic":
if action == "post" {
server.BadRequest(w, r, "imagetile '%s' can only PUT tiles not images", d.DataName())
return
}
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
if plane.ShapeDimensions() != 2 {
server.BadRequest(w, r, "Quadtrees can only return 2d images not %s", plane)
return
}
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
source, err := datastore.GetDataByUUID(uuid, d.Source)
if err != nil {
server.BadRequest(w, r, err)
return
}
src, ok := source.(*imageblk.Data)
if !ok {
server.BadRequest(w, r, "Cannot construct imagetile for non-voxels data: %s", d.Source)
return
}
img, err := d.GetImage(ctx, src, slice, parts[3] == "isotropic")
if err != nil {
server.BadRequest(w, r, err)
return
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: tile-accelerated %s %s (%s)", r.Method, planeStr, parts[3], r.URL)
default:
server.BadRequest(w, r, "Illegal request for imagetile data. See 'help' for REST API")
}
}
func (d *Data) ConstructTiles(uuidStr string, tileSpec TileSpec, request datastore.Request) error {
config := request.Settings()
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
return err
}
source, err := datastore.GetDataByUUID(uuid, d.Source)
if err != nil {
return err
}
src, ok := source.(*imageblk.Data)
if !ok {
return fmt.Errorf("Cannot construct imagetile for non-voxels data: %s", d.Source)
}
// Save the current tile specification
d.Levels = tileSpec
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
return err
}
// Get size of tile at lowest resolution.
lastLevel := Scaling(len(tileSpec) - 1)
loresSpec, found := tileSpec[lastLevel]
if !found {
return fmt.Errorf("Illegal tile spec. Should have levels 0 to absent %d.", lastLevel)
}
var loresSize [3]float64
for i := 0; i < 3; i++ {
loresSize[i] = float64(loresSpec.Resolution[i]) * float64(DefaultTileSize[i])
}
loresMag := dvid.Point3d{1, 1, 1}
for i := Scaling(0); i < lastLevel; i++ {
levelMag := tileSpec[i].levelMag
loresMag[0] *= levelMag[0]
loresMag[1] *= levelMag[1]
loresMag[2] *= levelMag[2]
}
// Get min and max points in terms of distance.
var minPtDist, maxPtDist [3]float64
for i := uint8(0); i < 3; i++ {
minPtDist[i] = float64(src.MinPoint.Value(i)) * float64(src.VoxelSize[i])
maxPtDist[i] = float64(src.MaxPoint.Value(i)) * float64(src.VoxelSize[i])
}
// Adjust min and max points for the tileable surface at lowest resolution.
var minTiledPt, maxTiledPt dvid.Point3d
for i := 0; i < 3; i++ {
minInt, _ := math.Modf(minPtDist[i] / loresSize[i])
maxInt, _ := math.Modf(maxPtDist[i] / loresSize[i])
minTileCoord := int32(minInt)
maxTileCoord := int32(maxInt)
minTiledPt[i] = minTileCoord * DefaultTileSize[i] * loresMag[i]
maxTiledPt[i] = (maxTileCoord+1)*DefaultTileSize[i]*loresMag[i] - 1
}
sizeVolume := maxTiledPt.Sub(minTiledPt).AddScalar(1)
// Setup swappable ExtData buffers (the stitched slices) so we can be generating tiles
// at same time we are reading and stitching them.
var bufferLock [2]sync.Mutex
var sliceBuffers [2]*imageblk.Voxels
var bufferNum int
// Get the planes we should tile.
planes, err := config.GetShapes("planes", ";")
if planes == nil {
// If no planes are specified, construct imagetile for 3 orthogonal planes.
planes = []dvid.DataShape{dvid.XY, dvid.XZ, dvid.YZ}
}
outF, err := d.putTileFunc(versionID)
// sort the tile spec keys to iterate from highest to lowest resolution
var sortedKeys []int
for scaling, _ := range tileSpec {
sortedKeys = append(sortedKeys, int(scaling))
}
sort.Ints(sortedKeys)
for _, plane := range planes {
timedLog := dvid.NewTimeLog()
offset := minTiledPt.Duplicate()
switch {
case plane.Equals(dvid.XY):
width, height, err := plane.GetSize2D(sizeVolume)
if err != nil {
return err
}
dvid.Debugf("Tiling XY image %d x %d pixels\n", width, height)
for z := src.MinPoint.Value(2); z <= src.MaxPoint.Value(2); z++ {
server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xy]")
sliceLog := dvid.NewTimeLog()
offset = offset.Modify(map[uint8]int32{2: z})
slice, err := dvid.NewOrthogSlice(dvid.XY, offset, dvid.Point2d{width, height})
if err != nil {
return err
}
bufferLock[bufferNum].Lock()
sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil)
if err != nil {
return err
}
if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil {
return err
}
// Iterate through the different scales, extracting tiles at each resolution.
go func(bufferNum int, offset dvid.Point) {
defer bufferLock[bufferNum].Unlock()
timedLog := dvid.NewTimeLog()
for _, key := range sortedKeys {
scaling := Scaling(key)
levelSpec := tileSpec[scaling]
if err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
if int(scaling) < len(tileSpec)-1 {
if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
}
}
timedLog.Debugf("Tiled XY Tile using buffer %d", bufferNum)
}(bufferNum, offset)
sliceLog.Infof("Read XY Tile @ Z = %d, now tiling...", z)
bufferNum = (bufferNum + 1) % 2
}
timedLog.Infof("Total time to generate XY Tiles")
case plane.Equals(dvid.XZ):
width, height, err := plane.GetSize2D(sizeVolume)
if err != nil {
return err
}
dvid.Debugf("Tiling XZ image %d x %d pixels\n", width, height)
for y := src.MinPoint.Value(1); y <= src.MaxPoint.Value(1); y++ {
server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xz]")
sliceLog := dvid.NewTimeLog()
offset = offset.Modify(map[uint8]int32{1: y})
slice, err := dvid.NewOrthogSlice(dvid.XZ, offset, dvid.Point2d{width, height})
if err != nil {
return err
}
bufferLock[bufferNum].Lock()
sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil)
if err != nil {
return err
}
if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil {
return err
}
// Iterate through the different scales, extracting tiles at each resolution.
go func(bufferNum int, offset dvid.Point) {
defer bufferLock[bufferNum].Unlock()
timedLog := dvid.NewTimeLog()
for _, key := range sortedKeys {
scaling := Scaling(key)
levelSpec := tileSpec[scaling]
if err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
if int(scaling) < len(tileSpec)-1 {
if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
}
}
timedLog.Debugf("Tiled XZ Tile using buffer %d", bufferNum)
}(bufferNum, offset)
sliceLog.Infof("Read XZ Tile @ Y = %d, now tiling...", y)
bufferNum = (bufferNum + 1) % 2
}
timedLog.Infof("Total time to generate XZ Tiles")
case plane.Equals(dvid.YZ):
width, height, err := plane.GetSize2D(sizeVolume)
if err != nil {
return err
}
dvid.Debugf("Tiling YZ image %d x %d pixels\n", width, height)
for x := src.MinPoint.Value(0); x <= src.MaxPoint.Value(0); x++ {
server.BlockOnInteractiveRequests("imagetile.ConstructTiles [yz]")
sliceLog := dvid.NewTimeLog()
offset = offset.Modify(map[uint8]int32{0: x})
slice, err := dvid.NewOrthogSlice(dvid.YZ, offset, dvid.Point2d{width, height})
if err != nil {
return err
}
bufferLock[bufferNum].Lock()
sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil)
if err != nil {
return err
}
if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil {
return err
}
// Iterate through the different scales, extracting tiles at each resolution.
go func(bufferNum int, offset dvid.Point) {
defer bufferLock[bufferNum].Unlock()
timedLog := dvid.NewTimeLog()
for _, key := range sortedKeys {
scaling := Scaling(key)
levelSpec := tileSpec[scaling]
outF, err := d.putTileFunc(versionID)
if err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
if int(scaling) < len(tileSpec)-1 {
if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil {
dvid.Errorf("Error in tiling: %v\n", err)
return
}
}
}
timedLog.Debugf("Tiled YZ Tile using buffer %d", bufferNum)
}(bufferNum, offset)
sliceLog.Debugf("Read YZ Tile @ X = %d, now tiling...", x)
bufferNum = (bufferNum + 1) % 2
}
timedLog.Infof("Total time to generate YZ Tiles")
default:
dvid.Infof("Skipping request to tile '%s'. Unsupported.", plane)
}
}
return nil
}
// GetBlocks returns a slice of bytes corresponding to all the blocks along a span in X
func (d *Data) GetBlocks(v dvid.VersionID, start dvid.ChunkPoint3d, span int32) ([]byte, error) {
timedLog := dvid.NewTimeLog()
defer timedLog.Infof("GetBlocks %s, span %d", start, span)
store, err := storage.MutableStore()
if err != nil {
return nil, fmt.Errorf("Data type imageblk had error initializing store: %v\n", err)
}
indexBeg := dvid.IndexZYX(start)
sx, sy, sz := indexBeg.Unpack()
end := start
end[0] += int32(span - 1)
indexEnd := dvid.IndexZYX(end)
keyBeg := NewTKey(&indexBeg)
keyEnd := NewTKey(&indexEnd)
// Allocate one uncompressed-sized slice with background values.
blockBytes := int32(d.BlockSize().Prod()) * d.Values.BytesPerElement()
numBytes := blockBytes * span
buf := make([]byte, numBytes, numBytes)
if d.Background != 0 {
for i := range buf {
buf[i] = byte(d.Background)
}
}
// Write the blocks that we can get concurrently on this byte slice.
ctx := datastore.NewVersionedCtx(d, v)
var wg sync.WaitGroup
err = store.ProcessRange(ctx, keyBeg, keyEnd, &storage.ChunkOp{}, func(c *storage.Chunk) error {
if c == nil || c.TKeyValue == nil {
return nil
}
kv := c.TKeyValue
if kv.V == nil {
return nil
}
// Determine which block this is.
indexZYX, err := DecodeTKey(kv.K)
if err != nil {
return err
}
x, y, z := indexZYX.Unpack()
if z != sz || y != sy || x < sx || x >= sx+int32(span) {
return fmt.Errorf("Received key-value for %s, not supposed to be within span range %s, length %d", *indexZYX, start, span)
}
n := x - sx
i := n * blockBytes
j := i + blockBytes
// Spawn goroutine to transfer data
wg.Add(1)
go xferBlock(buf[i:j], c, &wg)
return nil
})
if err != nil {
return nil, err
}
wg.Wait()
return buf, nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
action := strings.ToLower(r.Method)
switch action {
case "get", "post":
// Acceptable
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "incomplete API request")
return
}
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "metadata":
switch action {
case "post":
jsonBytes, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.SetMetadata(uuid, jsonBytes); err != nil {
server.BadRequest(w, r, err)
return
}
case "get":
if d.Levels == nil || len(d.Levels) == 0 {
server.BadRequest(w, r, "tile metadata for imagetile %q was not set\n", d.DataName())
return
}
metadata := struct {
MinTileCoord dvid.Point3d
MaxTileCoord dvid.Point3d
Levels TileSpec
}{
d.MinTileCoord,
d.MaxTileCoord,
d.Levels,
}
jsonBytes, err := json.Marshal(metadata)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
}
timedLog.Infof("HTTP %s: metadata (%s)", r.Method, r.URL)
case "tile":
switch action {
case "post":
err := d.PostTile(ctx, w, r, parts)
if err != nil {
server.BadRequest(w, r, "Error in posting tile with URL %q: %v\n", url, err)
return
}
case "get":
if err := d.ServeTile(ctx, w, r, parts); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL)
case "tilekey":
switch action {
case "get":
var err error
var hexkey string
if hexkey, err = d.GetTileKey(ctx, w, r, parts); err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"key": "%s"}`, hexkey)
timedLog.Infof("HTTP %s: tilekey (%s) returns %s", r.Method, r.URL, hexkey)
default:
server.BadRequest(w, r, fmt.Errorf("Cannot use HTTP %s for tilekey endpoint", action))
return
}
case "raw", "isotropic":
if action == "post" {
server.BadRequest(w, r, "imagetile '%s' can only PUT tiles not images", d.DataName())
return
}
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
if plane.ShapeDimensions() != 2 {
server.BadRequest(w, r, "Quadtrees can only return 2d images not %s", plane)
return
}
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
source, err := datastore.GetDataByUUIDName(uuid, d.Source)
if err != nil {
server.BadRequest(w, r, err)
return
}
src, ok := source.(*imageblk.Data)
if !ok {
server.BadRequest(w, r, "Cannot construct imagetile for non-voxels data: %s", d.Source)
return
}
img, err := d.GetImage(ctx, src, slice, parts[3] == "isotropic")
if err != nil {
server.BadRequest(w, r, err)
return
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: tile-accelerated %s %s (%s)", r.Method, planeStr, parts[3], r.URL)
default:
server.BadAPIRequest(w, r, d)
}
}
func (d *Data) foregroundROI(v dvid.VersionID, dest *roi.Data, background dvid.PointNd) {
dest.Ready = false
store, err := storage.MutableStore()
if err != nil {
dvid.Criticalf("Data type imageblk had error initializing store: %v\n", err)
return
}
timedLog := dvid.NewTimeLog()
timedLog.Infof("Starting foreground ROI %q for %s", dest.DataName(), d.DataName())
// Iterate through all voxel blocks, loading and then checking blocks
// for any foreground voxels.
ctx := datastore.NewVersionedCtx(d, v)
backgroundBytes := make([]byte, len(background))
for i, b := range background {
backgroundBytes[i] = byte(b)
}
const BATCH_SIZE = 1000
var numBatches int
var span *dvid.Span
spans := []dvid.Span{}
var f storage.ChunkFunc = func(chunk *storage.Chunk) error {
if chunk == nil || chunk.V == nil {
return nil
}
data, _, err := dvid.DeserializeData(chunk.V, true)
if err != nil {
return fmt.Errorf("Error decoding block: %v\n", err)
}
numVoxels := d.BlockSize().Prod()
var foreground bool
for i := int64(0); i < numVoxels; i++ {
isBackground := false
for _, b := range backgroundBytes {
if data[i] == b {
isBackground = true
break
}
}
if !isBackground {
foreground = true
break
}
}
if foreground {
indexZYX, err := DecodeTKey(chunk.K)
if err != nil {
return fmt.Errorf("Error decoding voxel block key: %v\n", err)
}
x, y, z := indexZYX.Unpack()
if span == nil {
span = &dvid.Span{z, y, x, x}
} else if !span.Extends(x, y, z) {
spans = append(spans, *span)
if len(spans) >= BATCH_SIZE {
init := (numBatches == 0)
numBatches++
go func(spans []dvid.Span) {
if err := dest.PutSpans(v, spans, init); err != nil {
dvid.Errorf("Error in storing ROI: %v\n", err)
} else {
timedLog.Debugf("-- Wrote batch %d of spans for foreground ROI %q", numBatches, dest.DataName())
}
}(spans)
spans = []dvid.Span{}
}
span = &dvid.Span{z, y, x, x}
}
}
server.BlockOnInteractiveRequests("voxels [compute foreground ROI]")
return nil
}
minTKey := storage.MinTKey(keyImageBlock)
maxTKey := storage.MaxTKey(keyImageBlock)
err = store.ProcessRange(ctx, minTKey, maxTKey, &storage.ChunkOp{}, f)
if err != nil {
dvid.Errorf("Error in processing chunks in ROI: %v\n", err)
return
}
if span != nil {
spans = append(spans, *span)
}
// Save new ROI
if len(spans) > 0 {
if err := dest.PutSpans(v, spans, numBatches == 0); err != nil {
dvid.Errorf("Error in storing ROI: %v\n", err)
return
}
}
timedLog.Infof("Created foreground ROI %q for %s", dest.DataName(), d.DataName())
dest.Ready = true
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "sync":
if action != "post" {
server.BadRequest(w, r, "Only POST allowed to sync endpoint")
return
}
replace := r.URL.Query().Get("replace") == "true"
if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil {
server.BadRequest(w, r, err)
return
}
case "count":
if action != "get" {
server.BadRequest(w, r, "Only GET action is available on 'count' endpoint.")
return
}
if len(parts) < 6 {
server.BadRequest(w, r, "Must include label and element type after 'count' endpoint.")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
i := StringToIndexType(parts[5])
if i == UnknownIndex {
server.BadRequest(w, r, fmt.Errorf("unknown index type specified (%q)", parts[5]))
return
}
count, err := d.GetCountElementType(ctx, label, i)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonStr := fmt.Sprintf(`{"Label":%d,%q:%d}`, label, i, count)
if _, err := io.WriteString(w, jsonStr); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: get count for label %d, index type %s: %s", r.Method, label, i, r.URL)
case "top":
if action != "get" {
server.BadRequest(w, r, "Only GET action is available on 'top' endpoint.")
return
}
if len(parts) < 6 {
server.BadRequest(w, r, "Must include N and element type after 'top' endpoint.")
return
}
n, err := strconv.ParseUint(parts[4], 10, 32)
if err != nil {
server.BadRequest(w, r, err)
return
}
i := StringToIndexType(parts[5])
if i == UnknownIndex {
server.BadRequest(w, r, fmt.Errorf("unknown index type specified (%q)", parts[5]))
return
}
labelSizes, err := d.GetTopElementType(ctx, int(n), i)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonBytes, err := json.Marshal(labelSizes)
if err != nil {
server.BadRequest(w, r, err)
return
}
if _, err := w.Write(jsonBytes); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: get top %d labels for index type %s: %s", r.Method, n, i, r.URL)
case "threshold":
if action != "get" {
server.BadRequest(w, r, "Only GET action is available on 'threshold' endpoint.")
return
}
if len(parts) < 6 {
server.BadRequest(w, r, "Must include threshold # and element type after 'threshold' endpoint.")
return
}
t, err := strconv.ParseUint(parts[4], 10, 32)
if err != nil {
server.BadRequest(w, r, err)
return
}
minSize := uint32(t)
i := StringToIndexType(parts[5])
if i == UnknownIndex {
server.BadRequest(w, r, fmt.Errorf("unknown index type specified (%q)", parts[5]))
return
}
queryStrings := r.URL.Query()
var num, offset int
offsetStr := queryStrings.Get("offset")
if offsetStr != "" {
offset, err = strconv.Atoi(offsetStr)
if err != nil {
server.BadRequest(w, r, fmt.Errorf("bad offset specified in query string (%q)", offsetStr))
return
}
}
numStr := queryStrings.Get("n")
if numStr != "" {
num, err = strconv.Atoi(numStr)
if err != nil {
server.BadRequest(w, r, fmt.Errorf("bad num specified in query string (%q)", numStr))
return
}
}
labels, err := d.GetLabelsByThreshold(ctx, i, minSize, offset, num)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonBytes, err := json.Marshal(labels)
if err != nil {
server.BadRequest(w, r, err)
return
}
if _, err := w.Write(jsonBytes); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: get %d labels for index type %s with threshold %d: %s", r.Method, num, i, t, r.URL)
default:
server.BadAPIRequest(w, r, d)
}
}
// Optimized bulk loading of XY images by loading all slices for a block before processing.
// Trades off memory for speed.
func (d *Data) loadXYImages(load *bulkLoadInfo) error {
// Load first slice, get dimensions, allocate blocks for whole slice.
// Note: We don't need to lock the block slices because goroutines do NOT
// access the same elements of a slice.
const numLayers = 2
var numBlocks int
var blocks [numLayers]storage.TKeyValues
var layerTransferred, layerWritten [numLayers]sync.WaitGroup
var waitForWrites sync.WaitGroup
curBlocks := 0
blockSize := d.BlockSize()
blockBytes := blockSize.Prod() * int64(d.Values.BytesPerElement())
// Iterate through XY slices batched into the Z length of blocks.
fileNum := 1
for _, filename := range load.filenames {
server.BlockOnInteractiveRequests("imageblk.loadXYImages")
timedLog := dvid.NewTimeLog()
zInBlock := load.offset.Value(2) % blockSize.Value(2)
firstSlice := fileNum == 1
lastSlice := fileNum == len(load.filenames)
firstSliceInBlock := firstSlice || zInBlock == 0
lastSliceInBlock := lastSlice || zInBlock == blockSize.Value(2)-1
lastBlocks := fileNum+int(blockSize.Value(2)) > len(load.filenames)
// Load images synchronously
vox, err := d.loadXYImage(filename, load.offset)
if err != nil {
return err
}
// Allocate blocks and/or load old block data if first/last XY blocks.
// Note: Slices are only zeroed out on first and last slice with assumption
// that ExtData is packed in XY footprint (values cover full extent).
// If that is NOT the case, we need to zero out blocks for each block layer.
if fileNum == 1 || (lastBlocks && firstSliceInBlock) {
numBlocks = dvid.GetNumBlocks(vox, blockSize)
if fileNum == 1 {
for layer := 0; layer < numLayers; layer++ {
blocks[layer] = make(storage.TKeyValues, numBlocks, numBlocks)
for b := 0; b < numBlocks; b++ {
blocks[layer][b].V = d.BackgroundBlock()
}
}
var bufSize uint64 = uint64(blockBytes) * uint64(numBlocks) * uint64(numLayers) / 1000000
dvid.Debugf("Allocated %d MB for buffers.\n", bufSize)
} else {
blocks[curBlocks] = make(storage.TKeyValues, numBlocks, numBlocks)
for b := 0; b < numBlocks; b++ {
blocks[curBlocks][b].V = d.BackgroundBlock()
}
}
err = d.loadOldBlocks(load.versionID, vox, blocks[curBlocks])
if err != nil {
return err
}
}
// Transfer data between external<->internal blocks asynchronously
layerTransferred[curBlocks].Add(1)
go func(vox *Voxels, curBlocks int) {
// Track point extents
if d.Extents().AdjustPoints(vox.StartPoint(), vox.EndPoint()) {
load.extentChanged.SetTrue()
}
// Process an XY image (slice).
changed, err := d.writeXYImage(load.versionID, vox, blocks[curBlocks])
if err != nil {
dvid.Infof("Error writing XY image: %v\n", err)
}
if changed {
load.extentChanged.SetTrue()
}
layerTransferred[curBlocks].Done()
}(vox, curBlocks)
// If this is the end of a block (or filenames), wait until all goroutines complete,
// then asynchronously write blocks.
if lastSliceInBlock {
waitForWrites.Add(1)
layerWritten[curBlocks].Add(1)
go func(curBlocks int) {
layerTransferred[curBlocks].Wait()
dvid.Debugf("Writing block buffer %d using %s and %s...\n",
curBlocks, d.Compression(), d.Checksum())
err := d.writeBlocks(load.versionID, blocks[curBlocks], &layerWritten[curBlocks], &waitForWrites)
if err != nil {
dvid.Errorf("Error in async write of voxel blocks: %v", err)
}
}(curBlocks)
// We can't move to buffer X until all blocks from buffer X have already been written.
curBlocks = (curBlocks + 1) % numLayers
dvid.Debugf("Waiting for layer %d to be written before reusing layer %d blocks\n",
curBlocks, curBlocks)
layerWritten[curBlocks].Wait()
dvid.Debugf("Using layer %d...\n", curBlocks)
}
fileNum++
load.offset = load.offset.Add(dvid.Point3d{0, 0, 1})
timedLog.Infof("Loaded %s slice %s", d.DataName(), vox)
}
waitForWrites.Wait()
return nil
}
// LoadLocal adds image data to a version node. See HelpMessage for example of
// command-line use of "load local".
func (d *Data) LoadLocal(request datastore.Request, reply *datastore.Response) error {
timedLog := dvid.NewTimeLog()
// Parse the request
var uuidStr, dataName, cmdStr, sourceStr, filename string
_ = request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr, &filename)
// Get the uuid from a uniquely identifiable string
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return fmt.Errorf("Could not find node with UUID %s: %v", uuidStr, err)
}
// Load the V3D Raw file.
ext := filepath.Ext(filename)
switch ext {
case ".raw", ".v3draw":
default:
return fmt.Errorf("Unknown extension '%s' when expected V3D Raw file", ext)
}
file, err := os.Open(filename)
if err != nil {
return err
}
unmarshaler := V3DRawMarshaler{}
channels, err := unmarshaler.UnmarshalV3DRaw(file)
if err != nil {
return err
}
// Store the metadata
d.NumChannels = len(channels)
d.Properties.Values = make(dvid.DataValues, d.NumChannels)
if d.NumChannels > 0 {
reply.Text = fmt.Sprintf("Loaded %s into data '%s': found %d channels\n",
d.DataName(), filename, d.NumChannels)
reply.Text += fmt.Sprintf(" %s", channels[0])
} else {
reply.Text = fmt.Sprintf("Found no channels in file %s\n", filename)
return nil
}
for i, channel := range channels {
d.Properties.Values[i] = channel.Voxels.Values()[0]
}
// Get repo and save it.
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
return err
}
// PUT each channel of the file into the datastore using a separate data name.
for _, channel := range channels {
dvid.Infof("Processing channel %d... \n", channel.channelNum)
err = d.PutVoxels(versionID, channel.Voxels, nil)
if err != nil {
return err
}
}
// Create a RGB composite from the first 3 channels. This is considered to be channel 0
// or can be accessed with the base data name.
dvid.Infof("Creating composite image from channels...\n")
err = d.storeComposite(versionID, channels)
if err != nil {
return err
}
timedLog.Infof("RPC load local '%s' completed", filename)
return nil
}
// PutLocal adds image data to a version node, altering underlying blocks if the image
// intersects the block.
//
// The image filename glob MUST BE absolute file paths that are visible to the server.
// This function is meant for mass ingestion of large data files, and it is inappropriate
// to read gigabytes of data just to send it over the network to a local DVID.
func (d *Data) PutLocal(request datastore.Request, reply *datastore.Response) error {
timedLog := dvid.NewTimeLog()
// Parse the request
var uuidStr, dataName, cmdStr, sourceStr, planeStr, offsetStr string
filenames := request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr,
&planeStr, &offsetStr)
if len(filenames) == 0 {
return fmt.Errorf("Need to include at least one file to add: %s", request)
}
// Get offset
offset, err := dvid.StringToPoint(offsetStr, ",")
if err != nil {
return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
}
// Get list of files to add
var addedFiles string
if len(filenames) == 1 {
addedFiles = filenames[0]
} else {
addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
}
dvid.Debugf(addedFiles + "\n")
// Get plane
plane, err := dvid.DataShapeString(planeStr).DataShape()
if err != nil {
return err
}
// Get Repo and IDs
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
// Load and PUT each image.
numSuccessful := 0
for _, filename := range filenames {
sliceLog := dvid.NewTimeLog()
img, _, err := dvid.GoImageFromFile(filename)
if err != nil {
return fmt.Errorf("Error after %d images successfully added: %v", numSuccessful, err)
}
slice, err := dvid.NewOrthogSlice(plane, offset, dvid.RectSize(img.Bounds()))
if err != nil {
return fmt.Errorf("Unable to determine slice: %v", err)
}
vox, err := d.NewVoxels(slice, img)
if err != nil {
return err
}
storage.FileBytesRead <- len(vox.Data())
if err = d.PutVoxels(versionID, vox, nil); err != nil {
return err
}
sliceLog.Debugf("%s put local %s", d.DataName(), slice)
numSuccessful++
offset = offset.Add(dvid.Point3d{0, 0, 1})
}
if err := datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
return err
}
timedLog.Infof("RPC put local (%s) completed", addedFiles)
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(ROI)
attenuationStr := queryValues.Get("attenuation")
if len(attenuationStr) != 0 {
attenuation, err := strconv.Atoi(attenuationStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if attenuation < 1 || attenuation > 7 {
server.BadRequest(w, r, "Attenuation should be from 1 to 7 (divides by 2^n)")
return
}
roiptr.attenuation = uint8(attenuation)
}
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
fmt.Printf("Setting configuration of data '%s'\n", d.DataName())
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
case "rawkey":
// GET <api URL>/node/<UUID>/<data name>/rawkey?x=<block x>&y=<block y>&z=<block z>
if len(parts) != 4 {
server.BadRequest(w, r, "rawkey endpoint should be followed by query strings (x, y, and z) giving block coord")
return
}
case "blocks":
// GET <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
// POST <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "%q must be followed by block-coord/span-x", parts[3])
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
data, err := d.GetBlocks(ctx.VersionID(), blockCoord, int32(span))
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
} else {
if err := d.PutBlocks(ctx.VersionID(), blockCoord, span, r.Body); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: Blocks (%s)", r.Method, r.URL)
case "arb":
// GET <api URL>/node/<UUID>/<data name>/arb/<top left>/<top right>/<bottom left>/<res>[/<format>]
if len(parts) < 8 {
server.BadRequest(w, r, "%q must be followed by top-left/top-right/bottom-left/res", parts[3])
return
}
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
img, err := d.GetArbitraryImage(ctx, parts[4], parts[5], parts[6], parts[7])
if err != nil {
server.BadRequest(w, r, err)
return
}
var formatStr string
if len(parts) >= 9 {
formatStr = parts[8]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: Arbitrary image (%s)", r.Method, r.URL)
case "raw", "isotropic":
// GET <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>]
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
if err != nil {
server.BadRequest(w, r, err)
return
}
vox, err := d.NewVoxels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), vox, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
if queryStrings.Get("throttle") == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
vox, err := d.NewVoxels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), vox, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
} else {
if isotropic {
err := fmt.Errorf("can only PUT 'raw' not 'isotropic' images")
server.BadRequest(w, r, err)
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store voxels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
vox, err := d.NewVoxels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), vox, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
}
default:
server.BadRequest(w, r, "Unrecognized API call for voxels %q. See API help.", d.DataName())
}
}
func (d *Data) serveTile(w http.ResponseWriter, r *http.Request, geom *GoogleSubvolGeom, formatStr string, noblanks bool) error {
// If it's outside, write blank tile unless user wants no blanks.
if geom.outside {
if noblanks {
http.NotFound(w, r)
return fmt.Errorf("Requested tile is outside of available volume.")
}
img, err := d.getBlankTileImage(geom)
if err != nil {
return err
}
return dvid.WriteImageHttp(w, img, formatStr)
}
// If we are within volume, get data from Google.
url, err := geom.GetURL(d.VolumeID, formatStr)
if err != nil {
return err
}
timedLog := dvid.NewTimeLog()
client, err := d.GetClient()
if err != nil {
dvid.Errorf("Can't get OAuth2 connection to Google: %v\n", err)
return err
}
resp, err := client.Get(url)
if err != nil {
return err
}
timedLog.Infof("PROXY HTTP to Google: %s, returned response %d", url, resp.StatusCode)
defer resp.Body.Close()
// Set the image header
if err := dvid.SetImageHeader(w, formatStr); err != nil {
return err
}
// If it's on edge, we need to pad the tile to the tile size.
if geom.edge {
// We need to read whole thing in to pad it.
data, err := ioutil.ReadAll(resp.Body)
timedLog.Infof("Got edge tile from Google, %d bytes\n", len(data))
if err != nil {
return err
}
paddedData, err := geom.padData(data)
if err != nil {
return err
}
_, err = w.Write(paddedData)
return err
}
// If we aren't on edge or outside, our return status should be OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Unexpected status code %d on tile request (%q, volume id %q)", resp.StatusCode, d.DataName(), d.VolumeID)
}
// Just send the data as we get it from Google in chunks.
respBytes := 0
const BufferSize = 32 * 1024
buf := make([]byte, BufferSize)
for {
n, err := resp.Body.Read(buf)
respBytes += n
eof := (err == io.EOF)
if err != nil && !eof {
return err
}
if _, err = w.Write(buf[:n]); err != nil {
return err
}
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
if eof {
break
}
}
timedLog.Infof("Got non-edge tile from Google, %d bytes\n", respBytes)
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
// TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints.
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
case "delete":
default:
server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *imageblk.ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(imageblk.ROI)
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "post" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "label":
// GET <api URL>/node/<UUID>/<data name>/label/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)
case "labels":
// POST <api URL>/node/<UUID>/<data name>/labels
if action != "post" {
server.BadRequest(w, r, "Batch labels query must be a POST request")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.")
return
}
var coords []dvid.Point3d
if err := json.Unmarshal(data, &coords); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "[")
sep := false
for _, coord := range coords {
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
if sep {
fmt.Fprintf(w, ",")
}
fmt.Fprintf(w, "%d", label)
sep = true
}
fmt.Fprintf(w, "]")
timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))
case "blocks":
// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "delete" {
server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
return
}
if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)
case "pseudocolor":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
lbl, err := d.NewLabels(slice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
// Convert to pseudocolor
pseudoColor, err := colorImage(img)
if err != nil {
server.BadRequest(w, r, err)
return
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
return
}
case "raw", "isotropic":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
lbl, err := d.NewLabels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
compression := queryValues.Get("compression")
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
lbl, err := d.NewLabels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
switch compression {
case "":
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "lz4":
compressed := make([]byte, lz4.CompressBound(data))
var n, outSize int
if outSize, err = lz4.Compress(data, compressed); err != nil {
server.BadRequest(w, r, err)
return
}
compressed = compressed[:outSize]
if n, err = w.Write(compressed); err != nil {
server.BadRequest(w, r, err)
return
}
if n != outSize {
errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
dvid.Errorf(errmsg)
server.BadRequest(w, r, errmsg)
return
}
case "gzip":
gw := gzip.NewWriter(w)
if _, err = gw.Write(data); err != nil {
server.BadRequest(w, r, err)
return
}
if err = gw.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
} else {
if isotropic {
server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
var data []byte
switch compression {
case "":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read 3d uncompressed POST")
case "lz4":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
if len(data) == 0 {
server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
return
}
tlog.Debugf("read 3d lz4 POST")
tlog = dvid.NewTimeLog()
uncompressed := make([]byte, subvol.NumVoxels()*8)
err = lz4.Uncompress(data, uncompressed)
if err != nil {
server.BadRequest(w, r, err)
return
}
data = uncompressed
tlog.Debugf("uncompressed 3d lz4 POST")
case "gzip":
tlog := dvid.NewTimeLog()
gr, err := gzip.NewReader(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err = ioutil.ReadAll(gr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err = gr.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read and uncompress 3d gzip POST")
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
lbl, err := d.NewLabels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
return
}
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.",
parts[3], d.DataName())
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
default:
}
// Get the data name and parse out the channel number or see if composite is required.
var channelNum int32
channumStr := strings.TrimPrefix(parts[2], string(d.DataName()))
if len(channumStr) == 0 {
channelNum = 0
} else {
n, err := strconv.ParseInt(channumStr, 10, 32)
if err != nil {
server.BadRequest(w, r, "Error parsing channel number from data name '%s': %v", parts[2], err)
return
}
if int(n) > d.NumChannels {
minChannelName := fmt.Sprintf("%s1", d.DataName())
maxChannelName := fmt.Sprintf("%s%d", d.DataName(), d.NumChannels)
server.BadRequest(w, r, "Data only has %d channels. Use names '%s' -> '%s'", d.NumChannels,
minChannelName, maxChannelName)
return
}
channelNum = int32(n)
}
// Get the data shape.
shapeStr := dvid.DataShapeString(parts[3])
dataShape, err := shapeStr.DataShape()
if err != nil {
server.BadRequest(w, r, "Bad data shape given '%s'", shapeStr)
return
}
switch dataShape.ShapeDimensions() {
case 2:
sizeStr, offsetStr := parts[4], parts[5]
slice, err := dvid.NewSliceFromStrings(shapeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "post" {
server.BadRequest(w, r, "DVID does not yet support POST of slices into multichannel data")
return
} else {
if d.NumChannels == 0 || d.Data.Values == nil {
server.BadRequest(w, r, "Cannot retrieve absent data '%d'. Please load data.", d.DataName())
return
}
values := d.Data.Values
if len(values) <= int(channelNum) {
server.BadRequest(w, r, "Must choose channel from 0 to %d", len(values))
return
}
stride := slice.Size().Value(0) * values.BytesPerElement()
dataValues := dvid.DataValues{values[channelNum]}
data := make([]uint8, int(slice.NumVoxels()))
v := imageblk.NewVoxels(slice, dataValues, data, stride)
channel := &Channel{
Voxels: v,
channelNum: channelNum,
}
img, err := d.GetImage(ctx.VersionID(), channel.Voxels, nil)
var formatStr string
if len(parts) >= 7 {
formatStr = parts[6]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
case 3:
sizeStr, offsetStr := parts[4], parts[5]
_, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
server.BadRequest(w, r, "DVID does not yet support GET of volume data")
return
} else {
server.BadRequest(w, r, "DVID does not yet support POST of volume data")
return
}
default:
server.BadRequest(w, r, "DVID does not yet support nD volumes")
return
}
timedLog.Infof("HTTP %s: %s", r.Method, dataShape)
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "incomplete API specification")
return
}
var comment string
action := strings.ToLower(r.Method)
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "info":
jsonStr, err := d.JSONString()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, jsonStr)
return
case "keys":
keyList, err := d.GetKeys(ctx)
if err != nil {
server.BadRequest(w, r, err)
return
}
jsonBytes, err := json.Marshal(keyList)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = "HTTP GET keys"
case "keyrange":
if len(parts) < 6 {
server.BadRequest(w, r, "expect beginning and end keys to follow 'keyrange' endpoint")
return
}
// Return JSON list of keys
keyBeg := parts[4]
keyEnd := parts[5]
keyList, err := d.GetKeysInRange(ctx, keyBeg, keyEnd)
if err != nil {
server.BadRequest(w, r, err)
return
}
jsonBytes, err := json.Marshal(keyList)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP GET keyrange [%q, %q]", keyBeg, keyEnd)
case "key":
if len(parts) < 5 {
server.BadRequest(w, r, "expect key string to follow 'key' endpoint")
return
}
keyStr := parts[4]
switch action {
case "get":
// Return value of single key
value, found, err := d.GetData(ctx, keyStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Key %q not found", keyStr), http.StatusNotFound)
return
}
if value != nil || len(value) > 0 {
_, err = w.Write(value)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
}
comment = fmt.Sprintf("HTTP GET key %q of keyvalue %q: %d bytes (%s)\n", keyStr, d.DataName(), len(value), url)
case "delete":
if err := d.DeleteData(ctx, keyStr); err != nil {
server.BadRequest(w, r, err)
return
}
comment = fmt.Sprintf("HTTP DELETE data with key %q of keyvalue %q (%s)\n", keyStr, d.DataName(), url)
case "post":
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
err = d.PutData(ctx, keyStr, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
comment = fmt.Sprintf("HTTP POST keyvalue '%s': %d bytes (%s)\n", d.DataName(), len(data), url)
default:
server.BadRequest(w, r, "key endpoint does not support %q HTTP verb", action)
return
}
default:
server.BadRequest(w, r, "unknown action %q requested", parts[3])
return
}
timedLog.Infof(comment)
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// versionID := ctx.VersionID()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "sync":
if action != "post" {
server.BadRequest(w, r, "Only POST allowed to sync endpoint")
return
}
replace := r.URL.Query().Get("replace") == "true"
if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil {
server.BadRequest(w, r, err)
return
}
case "label":
if action != "get" {
server.BadRequest(w, r, "Only GET action is available on 'label' endpoint.")
return
}
if len(parts) < 5 {
server.BadRequest(w, r, "Must include label after 'label' endpoint.")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
if label == 0 {
server.BadRequest(w, r, "Label 0 is protected background value and cannot be used for query.")
return
}
queryStrings := r.URL.Query()
jsonBytes, err := d.GetLabelJSON(ctx, label, queryStrings.Get("relationships") == "true")
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
if _, err := w.Write(jsonBytes); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: get synaptic elements for label %d (%s)", r.Method, label, r.URL)
case "tag":
if action != "get" {
server.BadRequest(w, r, "Only GET action is available on 'tag' endpoint.")
return
}
if len(parts) < 5 {
server.BadRequest(w, r, "Must include tag string after 'tag' endpoint.")
return
}
tag := Tag(parts[4])
queryStrings := r.URL.Query()
jsonBytes, err := d.GetTagJSON(ctx, tag, queryStrings.Get("relationships") == "true")
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
if _, err := w.Write(jsonBytes); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: get synaptic elements for tag %s (%s)", r.Method, tag, r.URL)
case "elements":
switch action {
case "get":
// GET <api URL>/node/<UUID>/<data name>/elements/<size>/<offset>
if len(parts) < 6 {
server.BadRequest(w, r, "Expect size and offset to follow 'elements' in GET request")
return
}
sizeStr, offsetStr := parts[4], parts[5]
ext3d, err := dvid.NewExtents3dFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
elements, err := d.GetRegionSynapses(ctx, ext3d)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonBytes, err := json.Marshal(elements)
if err != nil {
server.BadRequest(w, r, err)
return
}
if _, err := w.Write(jsonBytes); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: synapse elements in subvolume (size %s, offset %s) (%s)", r.Method, sizeStr, offsetStr, r.URL)
case "post":
if err := d.StoreSynapses(ctx, r.Body); err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "Only GET or POST action is available on 'elements' endpoint.")
return
}
case "element":
// DELETE <api URL>/node/<UUID>/<data name>/element/<coord>
if action != "delete" {
server.BadRequest(w, r, "Only DELETE action is available on 'element' endpoint.")
return
}
if len(parts) < 5 {
server.BadRequest(w, r, "Must include coordinate after DELETE on 'element' endpoint.")
return
}
pt, err := dvid.StringToPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.DeleteElement(ctx, pt); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: delete synaptic element at %s (%s)", r.Method, pt, r.URL)
case "move":
// POST <api URL>/node/<UUID>/<data name>/move/<from_coord>/<to_coord>
if action != "post" {
server.BadRequest(w, r, "Only POST action is available on 'move' endpoint.")
return
}
if len(parts) < 6 {
server.BadRequest(w, r, "Must include 'from' and 'to' coordinate after 'move' endpoint.")
return
}
fromPt, err := dvid.StringToPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
toPt, err := dvid.StringToPoint3d(parts[5], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.MoveElement(ctx, fromPt, toPt); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: move synaptic element from %s to %s (%s)", r.Method, fromPt, toPt, r.URL)
default:
server.BadAPIRequest(w, r, d)
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "incomplete API specification")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
default:
}
// Get the key and process request
var comment string
command := parts[3]
method := strings.ToLower(r.Method)
switch command {
case "roi":
switch method {
case "get":
if !d.Ready {
w.WriteHeader(http.StatusPartialContent)
}
jsonBytes, err := Get(ctx)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP GET ROI %q: %d bytes\n", d.DataName(), len(jsonBytes))
case "post":
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
err = d.PutJSON(ctx.VersionID(), data)
if err != nil {
server.BadRequest(w, r, err)
return
}
comment = fmt.Sprintf("HTTP POST ROI %q: %d bytes\n", d.DataName(), len(data))
case "delete":
if err := d.Delete(ctx); err != nil {
server.BadRequest(w, r, err)
return
}
comment = fmt.Sprintf("HTTP DELETE ROI %q\n", d.DataName())
}
case "mask":
if method != "get" {
server.BadRequest(w, r, "ROI mask only supports GET")
return
}
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", command)
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 3:
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err := d.GetMask(ctx, subvol)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "Currently only 3d masks ('0_1_2' shape) is supported")
return
}
case "ptquery":
switch method {
case "get":
server.BadRequest(w, r, "ptquery requires POST with list of points")
return
case "post":
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
jsonBytes, err := d.PointQuery(ctx, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP POST ptquery '%s'\n", d.DataName())
}
case "partition":
if method != "get" {
server.BadRequest(w, r, "partition only supports GET request")
return
}
queryValues := r.URL.Query()
batchsizeStr := queryValues.Get("batchsize")
batchsize, err := strconv.Atoi(batchsizeStr)
if err != nil {
server.BadRequest(w, r, fmt.Sprintf("Error reading batchsize query string: %v", err))
return
}
var jsonBytes []byte
optimizedStr := queryValues.Get("optimized")
dvid.Infof("queryvalues = %v\n", queryValues)
if optimizedStr == "true" || optimizedStr == "on" {
dvid.Infof("Perform optimized partitioning into subvolumes using batchsize %d\n", batchsize)
jsonBytes, err = d.Partition(ctx, int32(batchsize))
} else {
dvid.Infof("Performing simple partitioning into subvolumes using batchsize %d\n", batchsize)
jsonBytes, err = d.SimplePartition(ctx, int32(batchsize))
}
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
comment = fmt.Sprintf("HTTP partition '%s' with batch size %d\n",
d.DataName(), batchsize)
default:
w.Header().Set("Content-Type", "text/plain")
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
timedLog.Infof(comment)
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Check the action
action := strings.ToLower(r.Method)
if action != "get" {
server.BadRequest(w, r, "labelsurf data can only accept GET HTTP requests")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "surface":
// GET <api URL>/node/<UUID>/<data name>/surface/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'surface' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
fmt.Printf("Getting surface for label %d\n", label)
if err != nil {
server.BadRequest(w, r, err)
return
}
gzipData, found, err := GetSurface(ctx, label)
if err != nil {
server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
return
}
w.Header().Set("Content-type", "application/octet-stream")
if err := dvid.WriteGzip(gzipData, w, r); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: surface on label %d (%s)", r.Method, label, r.URL)
case "surface-by-point":
// GET <api URL>/node/<UUID>/<data name>/surface-by-point/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'surface-by-point' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
gzipData, found, err := GetSurface(ctx, label)
if err != nil {
server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
return
}
fmt.Printf("Found surface for label %d: %d bytes (gzip payload)\n", label, len(gzipData))
w.Header().Set("Content-type", "application/octet-stream")
if err := dvid.WriteGzip(gzipData, w, r); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: surface-by-point at %s (%s)", r.Method, coord, r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call '%s' for labelsurf data '%s'. See API help.",
parts[3], d.DataName())
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Check the action
action := strings.ToLower(r.Method)
if action != "get" {
server.BadRequest(w, r, "labelsz data can only accept GET HTTP requests")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "size":
// GET <api URL>/node/<UUID>/<data name>/size/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'size' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
size, err := d.GetSize(ctx.VersionID(), label)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "{%q: %d, %q: %d}", "Label", label, "Voxels", size)
timedLog.Infof("HTTP %s: get label %d size", r.Method, label)
case "sizerange":
// GET <api URL>/node/<UUID>/<data name>/sizerange/<min size>/<optional max size>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires at least the minimum size to follow 'sizerange' command")
return
}
minSize, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
var maxSize uint64
if len(parts) >= 6 {
maxSize, err = strconv.ParseUint(parts[5], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
jsonStr, err := d.GetSizeRange(ctx.VersionID(), minSize, maxSize)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: get labels with volume > %d and < %d (%s)", r.Method, minSize, maxSize, r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call '%s' for labelsz data '%s'. See API help.",
parts[3], d.DataName())
}
}
