func (d *Data) parseTileReq(r *http.Request, parts []string) (tileReq, error) {
if len(parts) < 7 {
return tileReq{}, fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
}
planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
// Construct the index for this tile
plane := dvid.DataShapeString(planeStr)
shape, err := plane.DataShape()
if err != nil {
err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
return tileReq{}, err
}
scaling, err := strconv.ParseUint(scalingStr, 10, 8)
if err != nil {
err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
return tileReq{}, err
}
tileCoord, err := dvid.StringToPoint(coordStr, "_")
if err != nil {
err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
return tileReq{}, err
}
if tileCoord.NumDims() != 3 {
err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr)
return tileReq{}, err
}
indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)}
return tileReq{shape, Scaling(scaling), indexZYX}, nil
}
// NewData returns a pointer to new ROI data with default values.
func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) {
basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
if err != nil {
return nil, err
}
s, found, err := c.GetString("BlockSize")
if err != nil {
return nil, err
}
var blockSize dvid.Point3d
if found {
pt, err := dvid.StringToPoint(s, ",")
if err != nil {
return nil, err
}
if pt.NumDims() != 3 {
return nil, fmt.Errorf("BlockSize must be 3d, not %dd", pt.NumDims())
}
blockSize, _ = pt.(dvid.Point3d)
} else {
blockSize = dvid.Point3d{DefaultBlockSize, DefaultBlockSize, DefaultBlockSize}
}
d := &Data{
Data: basedata,
Properties: Properties{blockSize, math.MaxInt32, math.MinInt32},
ready: make(map[dvid.VersionID]bool),
}
return d, nil
}
// DoRPC acts as a switchboard for RPC commands.
func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error {
switch req.TypeCommand() {
case "load":
if len(req.Command) < 5 {
return fmt.Errorf("Poorly formatted load command. See command-line help.")
}
// Parse the request
var uuidStr, dataName, cmdStr, offsetStr string
filenames, err := req.FilenameArgs(1, &uuidStr, &dataName, &cmdStr, &offsetStr)
if err != nil {
return err
}
if len(filenames) == 0 {
return fmt.Errorf("Need to include at least one file to add: %s", req)
}
// Get offset
offset, err := dvid.StringToPoint(offsetStr, ",")
if err != nil {
return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
}
// Get list of files to add
var addedFiles string
if len(filenames) == 1 {
addedFiles = filenames[0]
} else {
addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
}
dvid.Debugf(addedFiles + "\n")
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil {
return err
}
if err = d.LoadImages(versionID, offset, filenames); err != nil {
return err
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
return err
}
return nil
case "composite":
if len(req.Command) < 6 {
return fmt.Errorf("Poorly formatted composite command. See command-line help.")
}
return d.CreateComposite(req, reply)
default:
return fmt.Errorf("Unknown command. Data type '%s' [%s] does not support '%s' command.",
d.DataName(), d.TypeName(), req.TypeCommand())
}
return nil
}
// handleImageReq returns an image with appropriate Content-Type set. This function differs
// from handleTileReq in the way parameters are passed to it. handleTileReq accepts a tile coordinate.
// This function allows arbitrary offset and size, unconstrained by tile sizes.
func (d *Data) handleImageReq(w http.ResponseWriter, r *http.Request, parts []string) error {
if len(parts) < 7 {
return fmt.Errorf("%q must be followed by shape/size/offset", parts[3])
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
return err
}
var size dvid.Point
if size, err = dvid.StringToPoint(sizeStr, "_"); err != nil {
return err
}
offset, err := dvid.StringToPoint3d(offsetStr, "_")
if err != nil {
return err
}
// Determine how this request sits in the available scaled volumes.
scale, err := getScale(r)
if err != nil {
return err
}
geom, err := d.GetGoogleSubvolGeom(scale, plane, offset, size)
if err != nil {
return err
}
switch plane.ShapeDimensions() {
case 2:
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
if formatStr == "" {
formatStr = DefaultTileFormat
}
return d.serveTile(w, r, geom, formatStr, false)
case 3:
return d.serveVolume(w, r, geom, false)
}
return nil
}
// setByConfig sets Voxels properties based on type-specific keywords in the configuration.
// Any property not described in the config is left as is. See the Voxels help for listing
// of configurations.
func (p *Properties) setByConfig(config dvid.Config) error {
s, found, err := config.GetString("BlockSize")
if err != nil {
return err
}
if found {
p.BlockSize, err = dvid.StringToPoint(s, ",")
if err != nil {
return err
}
}
s, found, err = config.GetString("VoxelSize")
if err != nil {
return err
}
if found {
dvid.Infof("Changing resolution of voxels to %s\n", s)
p.Resolution.VoxelSize, err = dvid.StringToNdFloat32(s, ",")
if err != nil {
return err
}
}
s, found, err = config.GetString("VoxelUnits")
if err != nil {
return err
}
if found {
p.Resolution.VoxelUnits, err = dvid.StringToNdString(s, ",")
if err != nil {
return err
}
}
s, found, err = config.GetString("Background")
if err != nil {
return err
}
if found {
background, err := strconv.ParseUint(s, 10, 8)
if err != nil {
return err
}
p.Background = uint8(background)
}
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Check the action
action := strings.ToLower(r.Method)
if action != "get" {
server.BadRequest(w, r, "labelsurf data can only accept GET HTTP requests")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "surface":
// GET <api URL>/node/<UUID>/<data name>/surface/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'surface' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
fmt.Printf("Getting surface for label %d\n", label)
if err != nil {
server.BadRequest(w, r, err)
return
}
gzipData, found, err := GetSurface(ctx, label)
if err != nil {
server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
return
}
w.Header().Set("Content-type", "application/octet-stream")
if err := dvid.WriteGzip(gzipData, w, r); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: surface on label %d (%s)", r.Method, label, r.URL)
case "surface-by-point":
// GET <api URL>/node/<UUID>/<data name>/surface-by-point/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'surface-by-point' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
gzipData, found, err := GetSurface(ctx, label)
if err != nil {
server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
return
}
if !found {
http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
return
}
fmt.Printf("Found surface for label %d: %d bytes (gzip payload)\n", label, len(gzipData))
w.Header().Set("Content-type", "application/octet-stream")
if err := dvid.WriteGzip(gzipData, w, r); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: surface-by-point at %s (%s)", r.Method, coord, r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call '%s' for labelsurf data '%s'. See API help.",
parts[3], d.DataName())
}
}
// handleTileReq returns a tile with appropriate Content-Type set.
func (d *Data) handleTileReq(w http.ResponseWriter, r *http.Request, parts []string) error {
if len(parts) < 7 {
return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
}
planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
queryStrings := r.URL.Query()
var noblanks bool
noblanksStr := dvid.InstanceName(queryStrings.Get("noblanks"))
if noblanksStr == "true" {
noblanks = true
}
var tilesize int32 = DefaultTileSize
tileSizeStr := queryStrings.Get("tilesize")
if tileSizeStr != "" {
tilesizeInt, err := strconv.Atoi(tileSizeStr)
if err != nil {
return err
}
tilesize = int32(tilesizeInt)
}
size := dvid.Point2d{tilesize, tilesize}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
if formatStr == "" {
formatStr = DefaultTileFormat
}
// Parse the tile specification
plane := dvid.DataShapeString(planeStr)
shape, err := plane.DataShape()
if err != nil {
err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
server.BadRequest(w, r, err)
return err
}
scale, err := strconv.ParseUint(scalingStr, 10, 8)
if err != nil {
err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
server.BadRequest(w, r, err)
return err
}
tileCoord, err := dvid.StringToPoint(coordStr, "_")
if err != nil {
err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
server.BadRequest(w, r, err)
return err
}
// Convert tile coordinate to offset.
var ox, oy, oz int32
switch {
case shape.Equals(dvid.XY):
ox = tileCoord.Value(0) * tilesize
oy = tileCoord.Value(1) * tilesize
oz = tileCoord.Value(2)
case shape.Equals(dvid.XZ):
ox = tileCoord.Value(0) * tilesize
oy = tileCoord.Value(1)
oz = tileCoord.Value(2) * tilesize
case shape.Equals(dvid.YZ):
ox = tileCoord.Value(0)
oy = tileCoord.Value(1) * tilesize
oz = tileCoord.Value(2) * tilesize
default:
return fmt.Errorf("Unknown tile orientation: %s", shape)
}
// Determine how this request sits in the available scaled volumes.
geom, err := d.GetGoogleSubvolGeom(Scaling(scale), shape, dvid.Point3d{ox, oy, oz}, size)
if err != nil {
server.BadRequest(w, r, err)
return err
}
// Send the tile.
return d.serveTile(w, r, geom, formatStr, noblanks)
}
// ServeTile returns a tile with appropriate Content-Type set.
func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter,
r *http.Request, parts []string) error {
if len(parts) < 7 {
return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
}
planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
queryValues := r.URL.Query()
noblanksStr := dvid.InstanceName(queryValues.Get("noblanks"))
var noblanks bool
if noblanksStr == "true" {
noblanks = true
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
// Construct the index for this tile
plane := dvid.DataShapeString(planeStr)
shape, err := plane.DataShape()
if err != nil {
err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
server.BadRequest(w, r, err)
return err
}
scaling, err := strconv.ParseUint(scalingStr, 10, 8)
if err != nil {
err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
server.BadRequest(w, r, err)
return err
}
tileCoord, err := dvid.StringToPoint(coordStr, "_")
if err != nil {
err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
server.BadRequest(w, r, err)
return err
}
if tileCoord.NumDims() != 3 {
err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr)
server.BadRequest(w, r, err)
return err
}
indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)}
data, err := d.getTileData(ctx, shape, Scaling(scaling), indexZYX)
if err != nil {
server.BadRequest(w, r, err)
return err
}
if data == nil {
if noblanks {
http.NotFound(w, r)
return nil
}
img, err := d.getBlankTileImage(uuid, shape, Scaling(scaling))
if err != nil {
return err
}
return dvid.WriteImageHttp(w, img, formatStr)
}
switch d.Encoding {
case LZ4:
var img dvid.Image
if err := img.Deserialize(data); err != nil {
return err
}
data, err = img.GetPNG()
w.Header().Set("Content-type", "image/png")
case PNG:
w.Header().Set("Content-type", "image/png")
case JPG:
w.Header().Set("Content-type", "image/jpeg")
}
if err != nil {
server.BadRequest(w, r, err)
return err
}
if _, err = w.Write(data); err != nil {
return err
}
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
// TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints.
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
case "delete":
default:
server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *imageblk.ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(imageblk.ROI)
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "post" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "label":
// GET <api URL>/node/<UUID>/<data name>/label/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)
case "labels":
// POST <api URL>/node/<UUID>/<data name>/labels
if action != "post" {
server.BadRequest(w, r, "Batch labels query must be a POST request")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.")
return
}
var coords []dvid.Point3d
if err := json.Unmarshal(data, &coords); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "[")
sep := false
for _, coord := range coords {
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
if sep {
fmt.Fprintf(w, ",")
}
fmt.Fprintf(w, "%d", label)
sep = true
}
fmt.Fprintf(w, "]")
timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))
case "blocks":
// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "delete" {
server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
return
}
if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)
case "pseudocolor":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
lbl, err := d.NewLabels(slice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
// Convert to pseudocolor
pseudoColor, err := colorImage(img)
if err != nil {
server.BadRequest(w, r, err)
return
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
return
}
case "raw", "isotropic":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
lbl, err := d.NewLabels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
compression := queryValues.Get("compression")
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
lbl, err := d.NewLabels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
switch compression {
case "":
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "lz4":
compressed := make([]byte, lz4.CompressBound(data))
var n, outSize int
if outSize, err = lz4.Compress(data, compressed); err != nil {
server.BadRequest(w, r, err)
return
}
compressed = compressed[:outSize]
if n, err = w.Write(compressed); err != nil {
server.BadRequest(w, r, err)
return
}
if n != outSize {
errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
dvid.Errorf(errmsg)
server.BadRequest(w, r, errmsg)
return
}
case "gzip":
gw := gzip.NewWriter(w)
if _, err = gw.Write(data); err != nil {
server.BadRequest(w, r, err)
return
}
if err = gw.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
} else {
if isotropic {
server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
var data []byte
switch compression {
case "":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read 3d uncompressed POST")
case "lz4":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
if len(data) == 0 {
server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
return
}
tlog.Debugf("read 3d lz4 POST")
tlog = dvid.NewTimeLog()
uncompressed := make([]byte, subvol.NumVoxels()*8)
err = lz4.Uncompress(data, uncompressed)
if err != nil {
server.BadRequest(w, r, err)
return
}
data = uncompressed
tlog.Debugf("uncompressed 3d lz4 POST")
case "gzip":
tlog := dvid.NewTimeLog()
gr, err := gzip.NewReader(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err = ioutil.ReadAll(gr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err = gr.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read and uncompress 3d gzip POST")
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
lbl, err := d.NewLabels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
return
}
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.",
parts[3], d.DataName())
}
}
// PutLocal adds image data to a version node, altering underlying blocks if the image
// intersects the block.
//
// The image filename glob MUST BE absolute file paths that are visible to the server.
// This function is meant for mass ingestion of large data files, and it is inappropriate
// to read gigabytes of data just to send it over the network to a local DVID.
func (d *Data) PutLocal(request datastore.Request, reply *datastore.Response) error {
timedLog := dvid.NewTimeLog()
// Parse the request
var uuidStr, dataName, cmdStr, sourceStr, planeStr, offsetStr string
filenames := request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr,
&planeStr, &offsetStr)
if len(filenames) == 0 {
return fmt.Errorf("Need to include at least one file to add: %s", request)
}
// Get offset
offset, err := dvid.StringToPoint(offsetStr, ",")
if err != nil {
return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
}
// Get list of files to add
var addedFiles string
if len(filenames) == 1 {
addedFiles = filenames[0]
} else {
addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
}
dvid.Debugf(addedFiles + "\n")
// Get plane
plane, err := dvid.DataShapeString(planeStr).DataShape()
if err != nil {
return err
}
// Get Repo and IDs
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
// Load and PUT each image.
numSuccessful := 0
for _, filename := range filenames {
sliceLog := dvid.NewTimeLog()
img, _, err := dvid.GoImageFromFile(filename)
if err != nil {
return fmt.Errorf("Error after %d images successfully added: %v", numSuccessful, err)
}
slice, err := dvid.NewOrthogSlice(plane, offset, dvid.RectSize(img.Bounds()))
if err != nil {
return fmt.Errorf("Unable to determine slice: %v", err)
}
vox, err := d.NewVoxels(slice, img)
if err != nil {
return err
}
storage.FileBytesRead <- len(vox.Data())
if err = d.PutVoxels(versionID, vox, nil); err != nil {
return err
}
sliceLog.Debugf("%s put local %s", d.DataName(), slice)
numSuccessful++
offset = offset.Add(dvid.Point3d{0, 0, 1})
}
if err := datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
return err
}
timedLog.Infof("RPC put local (%s) completed", addedFiles)
return nil
}
// DoRPC acts as a switchboard for RPC commands.
func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error {
switch req.TypeCommand() {
case "load":
if len(req.Command) < 5 {
return fmt.Errorf("Poorly formatted load command. See command-line help.")
}
// Parse the request
var uuidStr, dataName, cmdStr, offsetStr string
filenames, err := req.FilenameArgs(1, &uuidStr, &dataName, &cmdStr, &offsetStr)
if err != nil {
return err
}
if len(filenames) == 0 {
hostname, _ := os.Hostname()
return fmt.Errorf("Couldn't find any files to add. Are they visible to DVID server on %s?",
hostname)
}
// Get offset
offset, err := dvid.StringToPoint(offsetStr, ",")
if err != nil {
return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
}
// Get list of files to add
var addedFiles string
if len(filenames) == 1 {
addedFiles = filenames[0]
} else {
addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
}
dvid.Debugf(addedFiles + "\n")
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil {
return err
}
return d.LoadImages(versionID, offset, filenames)
case "put":
if len(req.Command) < 7 {
return fmt.Errorf("Poorly formatted put command. See command-line help.")
}
source := req.Command[4]
switch source {
case "local":
return d.PutLocal(req, reply)
case "remote":
return fmt.Errorf("put remote not yet implemented")
default:
return fmt.Errorf("Unknown command. Data instance '%s' [%s] does not support '%s' command.",
d.DataName(), d.TypeName(), req.TypeCommand())
}
case "roi":
if len(req.Command) < 6 {
return fmt.Errorf("Poorly formatted roi command. See command-line help.")
}
return d.ForegroundROI(req, reply)
default:
return fmt.Errorf("Unknown command. Data instance '%s' [%s] does not support '%s' command.",
d.DataName(), d.TypeName(), req.TypeCommand())
}
return nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
versionID := ctx.VersionID()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "sparsevol":
// GET <api URL>/node/<UUID>/<data name>/sparsevol/<label>
// POST <api URL>/node/<UUID>/<data name>/sparsevol/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
switch action {
case "get":
queryValues := r.URL.Query()
var b Bounds
b.VoxelBounds, err = dvid.BoundsFromQueryString(r)
if err != nil {
server.BadRequest(w, r, "Error parsing bounds from query string: %v\n", err)
return
}
b.BlockBounds = b.VoxelBounds.Divide(d.BlockSize)
b.Exact = queryValues.Get("exact") == "true"
data, err := GetSparseVol(ctx, label, b)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "post":
server.BadRequest(w, r, "POST of sparsevol not currently implemented\n")
return
// if err := d.PutSparseVol(versionID, label, r.Body); err != nil {
// server.BadRequest(w, r, err)
// return
// }
default:
server.BadRequest(w, r, "Unable to handle HTTP action %s on sparsevol endpoint", action)
return
}
timedLog.Infof("HTTP %s: sparsevol on label %d (%s)", r.Method, label, r.URL)
case "sparsevol-by-point":
// GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'sparsevol-by-point' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
source, err := d.GetSyncedLabelblk(versionID)
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := source.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err := GetSparseVol(ctx, label, Bounds{})
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: sparsevol-by-point at %s (%s)", r.Method, coord, r.URL)
case "sparsevol-coarse":
// GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label>
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol-coarse' command")
return
}
label, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err := GetSparseCoarseVol(ctx, label)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: sparsevol-coarse on label %d (%s)", r.Method, label, r.URL)
case "maxlabel":
// GET <api URL>/node/<UUID>/<data name>/maxlabel
w.Header().Set("Content-Type", "application/json")
switch action {
case "get":
maxlabel, ok := d.MaxLabel[versionID]
if !ok {
server.BadRequest(w, r, "No maximum label found for %s version %d\n", d.DataName(), versionID)
return
}
fmt.Fprintf(w, "{%q: %d}", "maxlabel", maxlabel)
default:
server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
return
}
timedLog.Infof("HTTP maxlabel request (%s)", r.URL)
case "nextlabel":
// GET <api URL>/node/<UUID>/<data name>/nextlabel
// POST <api URL>/node/<UUID>/<data name>/nextlabel
w.Header().Set("Content-Type", "application/json")
switch action {
case "get":
fmt.Fprintf(w, "{%q: %d}", "nextlabel", d.MaxRepoLabel+1)
case "post":
server.BadRequest(w, r, "POST on maxlabel is not supported yet.\n")
return
default:
server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
return
}
timedLog.Infof("HTTP maxlabel request (%s)", r.URL)
case "split":
// POST <api URL>/node/<UUID>/<data name>/split/<label>
if action != "post" {
server.BadRequest(w, r, "Split requests must be POST actions.")
return
}
if len(parts) < 5 {
server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command")
return
}
fromLabel, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
server.BadRequest(w, r, err)
return
}
toLabel, err := d.SplitLabels(ctx.VersionID(), fromLabel, r.Body)
if err != nil {
server.BadRequest(w, r, fmt.Sprintf("Error on split: %v", err))
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "{%q: %d}", "label", toLabel)
timedLog.Infof("HTTP split request (%s)", r.URL)
case "merge":
// POST <api URL>/node/<UUID>/<data name>/merge
if action != "post" {
server.BadRequest(w, r, "Merge requests must be POST actions.")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for merge. Should be JSON.")
return
}
var tuple labels.MergeTuple
if err := json.Unmarshal(data, &tuple); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad merge op JSON: %v", err))
return
}
mergeOp, err := tuple.Op()
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.MergeLabels(ctx.VersionID(), mergeOp); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Error on merge: %v", err))
return
}
timedLog.Infof("HTTP merge request (%s)", r.URL)
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelvol data %q. See API help.",
parts[3], d.DataName())
}
}
