func (d *Data) parseTileReq(r *http.Request, parts []string) (tileReq, error) { if len(parts) < 7 { return tileReq{}, fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate") } planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6] // Construct the index for this tile plane := dvid.DataShapeString(planeStr) shape, err := plane.DataShape() if err != nil { err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err) return tileReq{}, err } scaling, err := strconv.ParseUint(scalingStr, 10, 8) if err != nil { err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err) return tileReq{}, err } tileCoord, err := dvid.StringToPoint(coordStr, "_") if err != nil { err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err) return tileReq{}, err } if tileCoord.NumDims() != 3 { err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr) return tileReq{}, err } indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)} return tileReq{shape, Scaling(scaling), indexZYX}, nil }
func (d *Data) ParseTileReq(r *http.Request, parts []string) (TileReq, error) { if len(parts) < 7 { return TileReq{}, fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate") } shapeStr, scalingStr, coordStr := parts[4], parts[5], parts[6] // Construct the index for this tile planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err) return TileReq{}, err } scale, err := strconv.ParseUint(scalingStr, 10, 8) if err != nil { err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err) return TileReq{}, err } tileCoord, err := dvid.StringToChunkPoint3d(coordStr, "_") if err != nil { err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err) return TileReq{}, err } return TileReq{tileCoord, plane, Scaling(scale)}, nil }
// ServeImage returns an image with appropriate Content-Type set. This function differs // from ServeTile in the way parameters are passed to it. ServeTile accepts a tile coordinate. // This function allows arbitrary offset and size, unconstrained by tile sizes. func (d *Data) ServeImage(w http.ResponseWriter, r *http.Request, parts []string) error { if len(parts) < 7 { return fmt.Errorf("%q must be followed by shape/size/offset", parts[3]) } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { return err } if plane.ShapeDimensions() != 2 { return fmt.Errorf("Quadtrees can only return 2d images not %s", plane) } size, err := dvid.StringToPoint2d(sizeStr, "_") if err != nil { return err } offset, err := dvid.StringToPoint3d(offsetStr, "_") if err != nil { return err } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } if formatStr == "" { formatStr = DefaultTileFormat } // See if scaling was specified in query string, otherwise use high-res (scale 0) var scale Scaling queryValues := r.URL.Query() scalingStr := queryValues.Get("scale") if scalingStr != "" { scale64, err := strconv.ParseUint(scalingStr, 10, 8) if err != nil { return fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err) } scale = Scaling(scale64) } // Determine how this request sits in the available scaled volumes. googleTile, err := d.GetGoogleSpec(scale, plane, offset, size) if err != nil { return err } // Send the tile. return d.serveTile(w, r, googleTile, formatStr, true) }
// handleImageReq returns an image with appropriate Content-Type set. This function differs // from handleTileReq in the way parameters are passed to it. handleTileReq accepts a tile coordinate. // This function allows arbitrary offset and size, unconstrained by tile sizes. func (d *Data) handleImageReq(w http.ResponseWriter, r *http.Request, parts []string) error { if len(parts) < 7 { return fmt.Errorf("%q must be followed by shape/size/offset", parts[3]) } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { return err } var size dvid.Point if size, err = dvid.StringToPoint(sizeStr, "_"); err != nil { return err } offset, err := dvid.StringToPoint3d(offsetStr, "_") if err != nil { return err } // Determine how this request sits in the available scaled volumes. scale, err := getScale(r) if err != nil { return err } geom, err := d.GetGoogleSubvolGeom(scale, plane, offset, size) if err != nil { return err } switch plane.ShapeDimensions() { case 2: var formatStr string if len(parts) >= 8 { formatStr = parts[7] } if formatStr == "" { formatStr = DefaultTileFormat } return d.serveTile(w, r, geom, formatStr, false) case 3: return d.serveVolume(w, r, geom, false) } return nil }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() // Get the action (GET, POST) action := strings.ToLower(r.Method) switch action { case "get": case "post": default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) return case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) return default: } // Get the data name and parse out the channel number or see if composite is required. var channelNum int32 channumStr := strings.TrimPrefix(parts[2], string(d.DataName())) if len(channumStr) == 0 { channelNum = 0 } else { n, err := strconv.ParseInt(channumStr, 10, 32) if err != nil { server.BadRequest(w, r, "Error parsing channel number from data name '%s': %v", parts[2], err) return } if int(n) > d.NumChannels { minChannelName := fmt.Sprintf("%s1", d.DataName()) maxChannelName := fmt.Sprintf("%s%d", d.DataName(), d.NumChannels) server.BadRequest(w, r, "Data only has %d channels. Use names '%s' -> '%s'", d.NumChannels, minChannelName, maxChannelName) return } channelNum = int32(n) } // Get the data shape. shapeStr := dvid.DataShapeString(parts[3]) dataShape, err := shapeStr.DataShape() if err != nil { server.BadRequest(w, r, "Bad data shape given '%s'", shapeStr) return } switch dataShape.ShapeDimensions() { case 2: sizeStr, offsetStr := parts[4], parts[5] slice, err := dvid.NewSliceFromStrings(shapeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action == "post" { server.BadRequest(w, r, "DVID does not yet support POST of slices into multichannel data") return } else { if d.NumChannels == 0 || d.Data.Values == nil { server.BadRequest(w, r, "Cannot retrieve absent data '%d'. Please load data.", d.DataName()) return } values := d.Data.Values if len(values) <= int(channelNum) { server.BadRequest(w, r, "Must choose channel from 0 to %d", len(values)) return } stride := slice.Size().Value(0) * values.BytesPerElement() dataValues := dvid.DataValues{values[channelNum]} data := make([]uint8, int(slice.NumVoxels())) v := imageblk.NewVoxels(slice, dataValues, data, stride) channel := &Channel{ Voxels: v, channelNum: channelNum, } img, err := d.GetImage(ctx.VersionID(), channel.Voxels, nil) var formatStr string if len(parts) >= 7 { formatStr = parts[6] } //dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice) err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } } case 3: sizeStr, offsetStr := parts[4], parts[5] _, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action == "get" { server.BadRequest(w, r, "DVID does not yet support GET of volume data") return } else { server.BadRequest(w, r, "DVID does not yet support POST of volume data") return } default: server.BadRequest(w, r, "DVID does not yet support nD volumes") return } timedLog.Infof("HTTP %s: %s", r.Method, dataShape) }
// handleTileReq returns a tile with appropriate Content-Type set. func (d *Data) handleTileReq(w http.ResponseWriter, r *http.Request, parts []string) error { if len(parts) < 7 { return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate") } planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6] queryStrings := r.URL.Query() var noblanks bool noblanksStr := dvid.InstanceName(queryStrings.Get("noblanks")) if noblanksStr == "true" { noblanks = true } var tilesize int32 = DefaultTileSize tileSizeStr := queryStrings.Get("tilesize") if tileSizeStr != "" { tilesizeInt, err := strconv.Atoi(tileSizeStr) if err != nil { return err } tilesize = int32(tilesizeInt) } size := dvid.Point2d{tilesize, tilesize} var formatStr string if len(parts) >= 8 { formatStr = parts[7] } if formatStr == "" { formatStr = DefaultTileFormat } // Parse the tile specification plane := dvid.DataShapeString(planeStr) shape, err := plane.DataShape() if err != nil { err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err) server.BadRequest(w, r, err) return err } scale, err := strconv.ParseUint(scalingStr, 10, 8) if err != nil { err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err) server.BadRequest(w, r, err) return err } tileCoord, err := dvid.StringToPoint(coordStr, "_") if err != nil { err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err) server.BadRequest(w, r, err) return err } // Convert tile coordinate to offset. var ox, oy, oz int32 switch { case shape.Equals(dvid.XY): ox = tileCoord.Value(0) * tilesize oy = tileCoord.Value(1) * tilesize oz = tileCoord.Value(2) case shape.Equals(dvid.XZ): ox = tileCoord.Value(0) * tilesize oy = tileCoord.Value(1) oz = tileCoord.Value(2) * tilesize case shape.Equals(dvid.YZ): ox = tileCoord.Value(0) oy = tileCoord.Value(1) * tilesize oz = tileCoord.Value(2) * tilesize default: return fmt.Errorf("Unknown tile orientation: %s", shape) } // Determine how this request sits in the available scaled volumes. geom, err := d.GetGoogleSubvolGeom(Scaling(scale), shape, dvid.Point3d{ox, oy, oz}, size) if err != nil { server.BadRequest(w, r, err) return err } // Send the tile. return d.serveTile(w, r, geom, formatStr, noblanks) }
// ServeTile returns a tile with appropriate Content-Type set. func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter, r *http.Request, parts []string) error { if len(parts) < 7 { return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate") } planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6] queryValues := r.URL.Query() noblanksStr := dvid.InstanceName(queryValues.Get("noblanks")) var noblanks bool if noblanksStr == "true" { noblanks = true } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } // Construct the index for this tile plane := dvid.DataShapeString(planeStr) shape, err := plane.DataShape() if err != nil { err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err) server.BadRequest(w, r, err) return err } scaling, err := strconv.ParseUint(scalingStr, 10, 8) if err != nil { err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err) server.BadRequest(w, r, err) return err } tileCoord, err := dvid.StringToPoint(coordStr, "_") if err != nil { err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err) server.BadRequest(w, r, err) return err } if tileCoord.NumDims() != 3 { err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr) server.BadRequest(w, r, err) return err } indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)} data, err := d.getTileData(ctx, shape, Scaling(scaling), indexZYX) if err != nil { server.BadRequest(w, r, err) return err } if data == nil { if noblanks { http.NotFound(w, r) return nil } img, err := d.getBlankTileImage(uuid, shape, Scaling(scaling)) if err != nil { return err } return dvid.WriteImageHttp(w, img, formatStr) } switch d.Encoding { case LZ4: var img dvid.Image if err := img.Deserialize(data); err != nil { return err } data, err = img.GetPNG() w.Header().Set("Content-type", "image/png") case PNG: w.Header().Set("Content-type", "image/png") case JPG: w.Header().Set("Content-type", "image/jpeg") } if err != nil { server.BadRequest(w, r, err) return err } if _, err = w.Write(data); err != nil { return err } return nil }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() action := strings.ToLower(r.Method) switch action { case "get", "post": // Acceptable default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } if len(parts) < 4 { server.BadRequest(w, r, "incomplete API request") return } switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "tile": if action == "post" { server.BadRequest(w, r, "DVID does not yet support POST of imagetile") return } if err := d.ServeTile(uuid, ctx, w, r, parts); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL) case "raw", "isotropic": if action == "post" { server.BadRequest(w, r, "imagetile '%s' can only PUT tiles not images", d.DataName()) return } if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3]) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } if plane.ShapeDimensions() != 2 { server.BadRequest(w, r, "Quadtrees can only return 2d images not %s", plane) return } slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { server.BadRequest(w, r, err) return } src, ok := source.(*imageblk.Data) if !ok { server.BadRequest(w, r, "Cannot construct imagetile for non-voxels data: %s", d.Source) return } img, err := d.GetImage(ctx, src, slice, parts[3] == "isotropic") if err != nil { server.BadRequest(w, r, err) return } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: tile-accelerated %s %s (%s)", r.Method, planeStr, parts[3], r.URL) default: server.BadRequest(w, r, "Illegal request for imagetile data. See 'help' for REST API") } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() action := strings.ToLower(r.Method) switch action { case "get", "post": // Acceptable default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } if len(parts) < 4 { server.BadRequest(w, r, "incomplete API request") return } switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "metadata": switch action { case "post": jsonBytes, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } if err := d.SetMetadata(uuid, jsonBytes); err != nil { server.BadRequest(w, r, err) return } case "get": if d.Levels == nil || len(d.Levels) == 0 { server.BadRequest(w, r, "tile metadata for imagetile %q was not set\n", d.DataName()) return } metadata := struct { MinTileCoord dvid.Point3d MaxTileCoord dvid.Point3d Levels TileSpec }{ d.MinTileCoord, d.MaxTileCoord, d.Levels, } jsonBytes, err := json.Marshal(metadata) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) } timedLog.Infof("HTTP %s: metadata (%s)", r.Method, r.URL) case "tile": switch action { case "post": err := d.PostTile(ctx, w, r, parts) if err != nil { server.BadRequest(w, r, "Error in posting tile with URL %q: %v\n", url, err) return } case "get": if err := d.ServeTile(ctx, w, r, parts); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL) case "tilekey": switch action { case "get": var err error var hexkey string if hexkey, err = d.GetTileKey(ctx, w, r, parts); err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, `{"key": "%s"}`, hexkey) timedLog.Infof("HTTP %s: tilekey (%s) returns %s", r.Method, r.URL, hexkey) default: server.BadRequest(w, r, fmt.Errorf("Cannot use HTTP %s for tilekey endpoint", action)) return } case "raw", "isotropic": if action == "post" { server.BadRequest(w, r, "imagetile '%s' can only PUT tiles not images", d.DataName()) return } if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3]) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } if plane.ShapeDimensions() != 2 { server.BadRequest(w, r, "Quadtrees can only return 2d images not %s", plane) return } slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } source, err := datastore.GetDataByUUIDName(uuid, d.Source) if err != nil { server.BadRequest(w, r, err) return } src, ok := source.(*imageblk.Data) if !ok { server.BadRequest(w, r, "Cannot construct imagetile for non-voxels data: %s", d.Source) return } img, err := d.GetImage(ctx, src, slice, parts[3] == "isotropic") if err != nil { server.BadRequest(w, r, err) return } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: tile-accelerated %s %s (%s)", r.Method, planeStr, parts[3], r.URL) default: server.BadAPIRequest(w, r, d) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { // TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints. timedLog := dvid.NewTimeLog() // Get the action (GET, POST) action := strings.ToLower(r.Method) switch action { case "get": case "post": case "delete": default: server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Get query strings and possible roi var roiptr *imageblk.ROI queryValues := r.URL.Query() roiname := dvid.InstanceName(queryValues.Get("roi")) if len(roiname) != 0 { roiptr = new(imageblk.ROI) } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "post" { config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, dtype.Help()) case "metadata": jsonStr, err := d.NdDataMetadata() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json") fmt.Fprintln(w, jsonStr) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "label": // GET <api URL>/node/<UUID>/<data name>/label/<coord> if len(parts) < 5 { server.BadRequest(w, r, "DVID requires coord to follow 'label' command") return } coord, err := dvid.StringToPoint(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } label, err := d.GetLabelAtPoint(ctx.VersionID(), coord) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") jsonStr := fmt.Sprintf(`{"Label": %d}`, label) fmt.Fprintf(w, jsonStr) timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL) case "labels": // POST <api URL>/node/<UUID>/<data name>/labels if action != "post" { server.BadRequest(w, r, "Batch labels query must be a POST request") return } data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.") return } var coords []dvid.Point3d if err := json.Unmarshal(data, &coords); err != nil { server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err)) return } w.Header().Set("Content-type", "application/json") fmt.Fprintf(w, "[") sep := false for _, coord := range coords { label, err := d.GetLabelAtPoint(ctx.VersionID(), coord) if err != nil { server.BadRequest(w, r, err) return } if sep { fmt.Fprintf(w, ",") } fmt.Fprintf(w, "%d", label) sep = true } fmt.Fprintf(w, "]") timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords)) case "blocks": // DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX> if len(parts) < 6 { server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.") return } blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } span, err := strconv.Atoi(parts[5]) if err != nil { server.BadRequest(w, r, err) return } if action != "delete" { server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint") return } if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL) case "pseudocolor": if len(parts) < 7 { server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3]) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 2: slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action != "get" { server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores") return } lbl, err := d.NewLabels(slice, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } img, err := d.GetImage(ctx.VersionID(), lbl, roiptr) if err != nil { server.BadRequest(w, r, err) return } // Convert to pseudocolor pseudoColor, err := colorImage(img) if err != nil { server.BadRequest(w, r, err) return } //dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice) var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, pseudoColor, formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL) default: server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests") return } case "raw", "isotropic": if len(parts) < 7 { server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3]) return } var isotropic bool = (parts[3] == "isotropic") shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 2: slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action != "get" { server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores") return } rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic) lbl, err := d.NewLabels(rawSlice, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } img, err := d.GetImage(ctx.VersionID(), lbl, roiptr) if err != nil { server.BadRequest(w, r, err) return } if isotropic { dstW := int(slice.Size().Value(0)) dstH := int(slice.Size().Value(1)) img, err = img.ScaleImage(dstW, dstH) if err != nil { server.BadRequest(w, r, err) return } } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } //dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice) err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL) case 3: queryStrings := r.URL.Query() throttle := queryStrings.Get("throttle") if throttle == "true" || throttle == "on" { select { case <-server.Throttle: // Proceed with operation, returning throttle token to server at end. defer func() { server.Throttle <- 1 }() default: throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations", server.MaxThrottledOps) http.Error(w, throttleMsg, http.StatusServiceUnavailable) return } } compression := queryValues.Get("compression") subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action == "get" { lbl, err := d.NewLabels(subvol, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") switch compression { case "": _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } case "lz4": compressed := make([]byte, lz4.CompressBound(data)) var n, outSize int if outSize, err = lz4.Compress(data, compressed); err != nil { server.BadRequest(w, r, err) return } compressed = compressed[:outSize] if n, err = w.Write(compressed); err != nil { server.BadRequest(w, r, err) return } if n != outSize { errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize) dvid.Errorf(errmsg) server.BadRequest(w, r, errmsg) return } case "gzip": gw := gzip.NewWriter(w) if _, err = gw.Write(data); err != nil { server.BadRequest(w, r, err) return } if err = gw.Close(); err != nil { server.BadRequest(w, r, err) return } default: server.BadRequest(w, r, "unknown compression type %q", compression) return } } else { if isotropic { server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images") return } // Make sure vox is block-aligned if !dvid.BlockAligned(subvol, d.BlockSize()) { server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint()) return } var data []byte switch compression { case "": tlog := dvid.NewTimeLog() data, err = ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } tlog.Debugf("read 3d uncompressed POST") case "lz4": tlog := dvid.NewTimeLog() data, err = ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } if len(data) == 0 { server.BadRequest(w, r, "received 0 LZ4 compressed bytes") return } tlog.Debugf("read 3d lz4 POST") tlog = dvid.NewTimeLog() uncompressed := make([]byte, subvol.NumVoxels()*8) err = lz4.Uncompress(data, uncompressed) if err != nil { server.BadRequest(w, r, err) return } data = uncompressed tlog.Debugf("uncompressed 3d lz4 POST") case "gzip": tlog := dvid.NewTimeLog() gr, err := gzip.NewReader(r.Body) if err != nil { server.BadRequest(w, r, err) return } data, err = ioutil.ReadAll(gr) if err != nil { server.BadRequest(w, r, err) return } if err = gr.Close(); err != nil { server.BadRequest(w, r, err) return } tlog.Debugf("read and uncompress 3d gzip POST") default: server.BadRequest(w, r, "unknown compression type %q", compression) return } lbl, err := d.NewLabels(subvol, data) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL) default: server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions") return } default: server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.", parts[3], d.DataName()) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } if len(parts) < 4 { server.BadRequest(w, r, "incomplete API specification") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) return case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) return default: } // Get the key and process request var comment string command := parts[3] method := strings.ToLower(r.Method) switch command { case "roi": switch method { case "get": if !d.Ready { w.WriteHeader(http.StatusPartialContent) } jsonBytes, err := Get(ctx) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) comment = fmt.Sprintf("HTTP GET ROI %q: %d bytes\n", d.DataName(), len(jsonBytes)) case "post": data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } err = d.PutJSON(ctx.VersionID(), data) if err != nil { server.BadRequest(w, r, err) return } comment = fmt.Sprintf("HTTP POST ROI %q: %d bytes\n", d.DataName(), len(data)) case "delete": if err := d.Delete(ctx); err != nil { server.BadRequest(w, r, err) return } comment = fmt.Sprintf("HTTP DELETE ROI %q\n", d.DataName()) } case "mask": if method != "get" { server.BadRequest(w, r, "ROI mask only supports GET") return } if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", command) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 3: subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } data, err := d.GetMask(ctx, subvol) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } default: server.BadRequest(w, r, "Currently only 3d masks ('0_1_2' shape) is supported") return } case "ptquery": switch method { case "get": server.BadRequest(w, r, "ptquery requires POST with list of points") return case "post": data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } jsonBytes, err := d.PointQuery(ctx, data) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) comment = fmt.Sprintf("HTTP POST ptquery '%s'\n", d.DataName()) } case "partition": if method != "get" { server.BadRequest(w, r, "partition only supports GET request") return } queryValues := r.URL.Query() batchsizeStr := queryValues.Get("batchsize") batchsize, err := strconv.Atoi(batchsizeStr) if err != nil { server.BadRequest(w, r, fmt.Sprintf("Error reading batchsize query string: %v", err)) return } var jsonBytes []byte optimizedStr := queryValues.Get("optimized") dvid.Infof("queryvalues = %v\n", queryValues) if optimizedStr == "true" || optimizedStr == "on" { dvid.Infof("Perform optimized partitioning into subvolumes using batchsize %d\n", batchsize) jsonBytes, err = d.Partition(ctx, int32(batchsize)) } else { dvid.Infof("Performing simple partitioning into subvolumes using batchsize %d\n", batchsize) jsonBytes, err = d.SimplePartition(ctx, int32(batchsize)) } if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) comment = fmt.Sprintf("HTTP partition '%s' with batch size %d\n", d.DataName(), batchsize) default: w.Header().Set("Content-Type", "text/plain") server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } timedLog.Infof(comment) }
// PutLocal adds image data to a version node, altering underlying blocks if the image // intersects the block. // // The image filename glob MUST BE absolute file paths that are visible to the server. // This function is meant for mass ingestion of large data files, and it is inappropriate // to read gigabytes of data just to send it over the network to a local DVID. func (d *Data) PutLocal(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, sourceStr, planeStr, offsetStr string filenames := request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr, &planeStr, &offsetStr) if len(filenames) == 0 { return fmt.Errorf("Need to include at least one file to add: %s", request) } // Get offset offset, err := dvid.StringToPoint(offsetStr, ",") if err != nil { return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err) } // Get list of files to add var addedFiles string if len(filenames) == 1 { addedFiles = filenames[0] } else { addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1) } dvid.Debugf(addedFiles + "\n") // Get plane plane, err := dvid.DataShapeString(planeStr).DataShape() if err != nil { return err } // Get Repo and IDs uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Load and PUT each image. numSuccessful := 0 for _, filename := range filenames { sliceLog := dvid.NewTimeLog() img, _, err := dvid.GoImageFromFile(filename) if err != nil { return fmt.Errorf("Error after %d images successfully added: %v", numSuccessful, err) } slice, err := dvid.NewOrthogSlice(plane, offset, dvid.RectSize(img.Bounds())) if err != nil { return fmt.Errorf("Unable to determine slice: %v", err) } vox, err := d.NewVoxels(slice, img) if err != nil { return err } storage.FileBytesRead <- len(vox.Data()) if err = d.PutVoxels(versionID, vox, nil); err != nil { return err } sliceLog.Debugf("%s put local %s", d.DataName(), slice) numSuccessful++ offset = offset.Add(dvid.Point3d{0, 0, 1}) } if err := datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } timedLog.Infof("RPC put local (%s) completed", addedFiles) return nil }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() // Get the action (GET, POST) action := strings.ToLower(r.Method) switch action { case "get": case "post": default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Get query strings and possible roi var roiptr *ROI queryValues := r.URL.Query() roiname := dvid.InstanceName(queryValues.Get("roi")) if len(roiname) != 0 { roiptr = new(ROI) attenuationStr := queryValues.Get("attenuation") if len(attenuationStr) != 0 { attenuation, err := strconv.Atoi(attenuationStr) if err != nil { server.BadRequest(w, r, err) return } if attenuation < 1 || attenuation > 7 { server.BadRequest(w, r, "Attenuation should be from 1 to 7 (divides by 2^n)") return } roiptr.attenuation = uint8(attenuation) } } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "put" { fmt.Printf("Setting configuration of data '%s'\n", d.DataName()) config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) return case "metadata": jsonStr, err := d.NdDataMetadata() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json") fmt.Fprintln(w, jsonStr) return case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) return case "rawkey": // GET <api URL>/node/<UUID>/<data name>/rawkey?x=<block x>&y=<block y>&z=<block z> if len(parts) != 4 { server.BadRequest(w, r, "rawkey endpoint should be followed by query strings (x, y, and z) giving block coord") return } case "blocks": // GET <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX> // POST <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX> if len(parts) < 6 { server.BadRequest(w, r, "%q must be followed by block-coord/span-x", parts[3]) return } blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } span, err := strconv.Atoi(parts[5]) if err != nil { server.BadRequest(w, r, err) return } if action == "get" { data, err := d.GetBlocks(ctx.VersionID(), blockCoord, int32(span)) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } } else { if err := d.PutBlocks(ctx.VersionID(), blockCoord, span, r.Body); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: Blocks (%s)", r.Method, r.URL) case "arb": // GET <api URL>/node/<UUID>/<data name>/arb/<top left>/<top right>/<bottom left>/<res>[/<format>] if len(parts) < 8 { server.BadRequest(w, r, "%q must be followed by top-left/top-right/bottom-left/res", parts[3]) return } queryStrings := r.URL.Query() throttle := queryStrings.Get("throttle") if throttle == "true" || throttle == "on" { select { case <-server.Throttle: // Proceed with operation, returning throttle token to server at end. defer func() { server.Throttle <- 1 }() default: throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations", server.MaxThrottledOps) http.Error(w, throttleMsg, http.StatusServiceUnavailable) return } } img, err := d.GetArbitraryImage(ctx, parts[4], parts[5], parts[6], parts[7]) if err != nil { server.BadRequest(w, r, err) return } var formatStr string if len(parts) >= 9 { formatStr = parts[8] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: Arbitrary image (%s)", r.Method, r.URL) case "raw", "isotropic": // GET <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>] if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3]) return } var isotropic bool = (parts[3] == "isotropic") shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 2: slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action != "get" { server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores") return } rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic) if err != nil { server.BadRequest(w, r, err) return } vox, err := d.NewVoxels(rawSlice, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox) if err != nil { server.BadRequest(w, r, err) return } } img, err := d.GetImage(ctx.VersionID(), vox, roiptr) if err != nil { server.BadRequest(w, r, err) return } if isotropic { dstW := int(slice.Size().Value(0)) dstH := int(slice.Size().Value(1)) img, err = img.ScaleImage(dstW, dstH) if err != nil { server.BadRequest(w, r, err) return } } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL) case 3: queryStrings := r.URL.Query() if queryStrings.Get("throttle") == "on" { select { case <-server.Throttle: // Proceed with operation, returning throttle token to server at end. defer func() { server.Throttle <- 1 }() default: throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations", server.MaxThrottledOps) http.Error(w, throttleMsg, http.StatusServiceUnavailable) return } } subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action == "get" { vox, err := d.NewVoxels(subvol, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox) if err != nil { server.BadRequest(w, r, err) return } } data, err := d.GetVolume(ctx.VersionID(), vox, roiptr) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } } else { if isotropic { err := fmt.Errorf("can only PUT 'raw' not 'isotropic' images") server.BadRequest(w, r, err) return } // Make sure vox is block-aligned if !dvid.BlockAligned(subvol, d.BlockSize()) { server.BadRequest(w, r, "cannot store voxels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint()) return } data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } vox, err := d.NewVoxels(subvol, data) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox) if err != nil { server.BadRequest(w, r, err) return } } if err = d.PutVoxels(ctx.VersionID(), vox, roiptr); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL) default: server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions") } default: server.BadRequest(w, r, "Unrecognized API call for voxels %q. See API help.", d.DataName()) } }