Example #1
0
func (d *Data) parseTileReq(r *http.Request, parts []string) (tileReq, error) {
	if len(parts) < 7 {
		return tileReq{}, fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
	}
	planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]

	// Construct the index for this tile
	plane := dvid.DataShapeString(planeStr)
	shape, err := plane.DataShape()
	if err != nil {
		err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
		return tileReq{}, err
	}
	scaling, err := strconv.ParseUint(scalingStr, 10, 8)
	if err != nil {
		err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
		return tileReq{}, err
	}
	tileCoord, err := dvid.StringToPoint(coordStr, "_")
	if err != nil {
		err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
		return tileReq{}, err
	}
	if tileCoord.NumDims() != 3 {
		err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr)
		return tileReq{}, err
	}
	indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)}
	return tileReq{shape, Scaling(scaling), indexZYX}, nil
}
Example #2
0
// NewData returns a pointer to new ROI data with default values.
func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) {
	basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
	if err != nil {
		return nil, err
	}
	s, found, err := c.GetString("BlockSize")
	if err != nil {
		return nil, err
	}
	var blockSize dvid.Point3d
	if found {
		pt, err := dvid.StringToPoint(s, ",")
		if err != nil {
			return nil, err
		}
		if pt.NumDims() != 3 {
			return nil, fmt.Errorf("BlockSize must be 3d, not %dd", pt.NumDims())
		}
		blockSize, _ = pt.(dvid.Point3d)
	} else {
		blockSize = dvid.Point3d{DefaultBlockSize, DefaultBlockSize, DefaultBlockSize}
	}
	d := &Data{
		Data:       basedata,
		Properties: Properties{blockSize, math.MaxInt32, math.MinInt32},
		ready:      make(map[dvid.VersionID]bool),
	}
	return d, nil
}
Example #3
0
// DoRPC acts as a switchboard for RPC commands.
func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error {
	switch req.TypeCommand() {
	case "load":
		if len(req.Command) < 5 {
			return fmt.Errorf("Poorly formatted load command.  See command-line help.")
		}
		// Parse the request
		var uuidStr, dataName, cmdStr, offsetStr string
		filenames, err := req.FilenameArgs(1, &uuidStr, &dataName, &cmdStr, &offsetStr)
		if err != nil {
			return err
		}
		if len(filenames) == 0 {
			return fmt.Errorf("Need to include at least one file to add: %s", req)
		}

		// Get offset
		offset, err := dvid.StringToPoint(offsetStr, ",")
		if err != nil {
			return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
		}

		// Get list of files to add
		var addedFiles string
		if len(filenames) == 1 {
			addedFiles = filenames[0]
		} else {
			addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
		}
		dvid.Debugf(addedFiles + "\n")

		uuid, versionID, err := datastore.MatchingUUID(uuidStr)
		if err != nil {
			return err
		}
		if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil {
			return err
		}
		if err = d.LoadImages(versionID, offset, filenames); err != nil {
			return err
		}
		if err := datastore.SaveDataByUUID(uuid, d); err != nil {
			return err
		}
		return nil

	case "composite":
		if len(req.Command) < 6 {
			return fmt.Errorf("Poorly formatted composite command.  See command-line help.")
		}
		return d.CreateComposite(req, reply)

	default:
		return fmt.Errorf("Unknown command.  Data type '%s' [%s] does not support '%s' command.",
			d.DataName(), d.TypeName(), req.TypeCommand())
	}
	return nil
}
Example #4
0
// handleImageReq returns an image with appropriate Content-Type set.  This function differs
// from handleTileReq in the way parameters are passed to it.  handleTileReq accepts a tile coordinate.
// This function allows arbitrary offset and size, unconstrained by tile sizes.
func (d *Data) handleImageReq(w http.ResponseWriter, r *http.Request, parts []string) error {
	if len(parts) < 7 {
		return fmt.Errorf("%q must be followed by shape/size/offset", parts[3])
	}
	shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
	planeStr := dvid.DataShapeString(shapeStr)
	plane, err := planeStr.DataShape()
	if err != nil {
		return err
	}

	var size dvid.Point
	if size, err = dvid.StringToPoint(sizeStr, "_"); err != nil {
		return err
	}
	offset, err := dvid.StringToPoint3d(offsetStr, "_")
	if err != nil {
		return err
	}

	// Determine how this request sits in the available scaled volumes.
	scale, err := getScale(r)
	if err != nil {
		return err
	}
	geom, err := d.GetGoogleSubvolGeom(scale, plane, offset, size)
	if err != nil {
		return err
	}

	switch plane.ShapeDimensions() {
	case 2:
		var formatStr string
		if len(parts) >= 8 {
			formatStr = parts[7]
		}
		if formatStr == "" {
			formatStr = DefaultTileFormat
		}

		return d.serveTile(w, r, geom, formatStr, false)
	case 3:
		return d.serveVolume(w, r, geom, false)
	}
	return nil
}
Example #5
0
// setByConfig sets Voxels properties based on type-specific keywords in the configuration.
// Any property not described in the config is left as is.  See the Voxels help for listing
// of configurations.
func (p *Properties) setByConfig(config dvid.Config) error {
	s, found, err := config.GetString("BlockSize")
	if err != nil {
		return err
	}
	if found {
		p.BlockSize, err = dvid.StringToPoint(s, ",")
		if err != nil {
			return err
		}
	}
	s, found, err = config.GetString("VoxelSize")
	if err != nil {
		return err
	}
	if found {
		dvid.Infof("Changing resolution of voxels to %s\n", s)
		p.Resolution.VoxelSize, err = dvid.StringToNdFloat32(s, ",")
		if err != nil {
			return err
		}
	}
	s, found, err = config.GetString("VoxelUnits")
	if err != nil {
		return err
	}
	if found {
		p.Resolution.VoxelUnits, err = dvid.StringToNdString(s, ",")
		if err != nil {
			return err
		}
	}
	s, found, err = config.GetString("Background")
	if err != nil {
		return err
	}
	if found {
		background, err := strconv.ParseUint(s, 10, 8)
		if err != nil {
			return err
		}
		p.Background = uint8(background)
	}
	return nil
}
Example #6
0
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
	timedLog := dvid.NewTimeLog()

	// Check the action
	action := strings.ToLower(r.Method)
	if action != "get" {
		server.BadRequest(w, r, "labelsurf data can only accept GET HTTP requests")
		return
	}

	// Break URL request into arguments
	url := r.URL.Path[len(server.WebAPIPath):]
	parts := strings.Split(url, "/")
	if len(parts[len(parts)-1]) == 0 {
		parts = parts[:len(parts)-1]
	}

	if len(parts) < 4 {
		server.BadRequest(w, r, "Incomplete API request")
		return
	}

	// Process help and info.
	switch parts[3] {
	case "help":
		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprintln(w, dtype.Help())

	case "info":
		jsonBytes, err := d.MarshalJSON()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, string(jsonBytes))

	case "surface":
		// GET <api URL>/node/<UUID>/<data name>/surface/<label>
		if len(parts) < 5 {
			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'surface' command")
			return
		}
		label, err := strconv.ParseUint(parts[4], 10, 64)
		fmt.Printf("Getting surface for label %d\n", label)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		gzipData, found, err := GetSurface(ctx, label)
		if err != nil {
			server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
			return
		}
		if !found {
			http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
			return
		}
		w.Header().Set("Content-type", "application/octet-stream")
		if err := dvid.WriteGzip(gzipData, w, r); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: surface on label %d (%s)", r.Method, label, r.URL)

	case "surface-by-point":
		// GET <api URL>/node/<UUID>/<data name>/surface-by-point/<coord>
		if len(parts) < 5 {
			server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'surface-by-point' command")
			return
		}
		coord, err := dvid.StringToPoint(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		gzipData, found, err := GetSurface(ctx, label)
		if err != nil {
			server.BadRequest(w, r, "Error on getting surface for label %d: %v", label, err)
			return
		}
		if !found {
			http.Error(w, fmt.Sprintf("Surface for label '%d' not found", label), http.StatusNotFound)
			return
		}
		fmt.Printf("Found surface for label %d: %d bytes (gzip payload)\n", label, len(gzipData))
		w.Header().Set("Content-type", "application/octet-stream")
		if err := dvid.WriteGzip(gzipData, w, r); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: surface-by-point at %s (%s)", r.Method, coord, r.URL)

	default:
		server.BadRequest(w, r, "Unrecognized API call '%s' for labelsurf data '%s'.  See API help.",
			parts[3], d.DataName())
	}
}
Example #7
0
// handleTileReq returns a tile with appropriate Content-Type set.
func (d *Data) handleTileReq(w http.ResponseWriter, r *http.Request, parts []string) error {

	if len(parts) < 7 {
		return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
	}
	planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
	queryStrings := r.URL.Query()

	var noblanks bool
	noblanksStr := dvid.InstanceName(queryStrings.Get("noblanks"))
	if noblanksStr == "true" {
		noblanks = true
	}

	var tilesize int32 = DefaultTileSize
	tileSizeStr := queryStrings.Get("tilesize")
	if tileSizeStr != "" {
		tilesizeInt, err := strconv.Atoi(tileSizeStr)
		if err != nil {
			return err
		}
		tilesize = int32(tilesizeInt)
	}
	size := dvid.Point2d{tilesize, tilesize}

	var formatStr string
	if len(parts) >= 8 {
		formatStr = parts[7]
	}
	if formatStr == "" {
		formatStr = DefaultTileFormat
	}

	// Parse the tile specification
	plane := dvid.DataShapeString(planeStr)
	shape, err := plane.DataShape()
	if err != nil {
		err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	scale, err := strconv.ParseUint(scalingStr, 10, 8)
	if err != nil {
		err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	tileCoord, err := dvid.StringToPoint(coordStr, "_")
	if err != nil {
		err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
		server.BadRequest(w, r, err)
		return err
	}

	// Convert tile coordinate to offset.
	var ox, oy, oz int32
	switch {
	case shape.Equals(dvid.XY):
		ox = tileCoord.Value(0) * tilesize
		oy = tileCoord.Value(1) * tilesize
		oz = tileCoord.Value(2)
	case shape.Equals(dvid.XZ):
		ox = tileCoord.Value(0) * tilesize
		oy = tileCoord.Value(1)
		oz = tileCoord.Value(2) * tilesize
	case shape.Equals(dvid.YZ):
		ox = tileCoord.Value(0)
		oy = tileCoord.Value(1) * tilesize
		oz = tileCoord.Value(2) * tilesize
	default:
		return fmt.Errorf("Unknown tile orientation: %s", shape)
	}

	// Determine how this request sits in the available scaled volumes.
	geom, err := d.GetGoogleSubvolGeom(Scaling(scale), shape, dvid.Point3d{ox, oy, oz}, size)
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}

	// Send the tile.
	return d.serveTile(w, r, geom, formatStr, noblanks)
}
Example #8
0
// ServeTile returns a tile with appropriate Content-Type set.
func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter,
	r *http.Request, parts []string) error {

	if len(parts) < 7 {
		return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
	}
	planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
	queryValues := r.URL.Query()
	noblanksStr := dvid.InstanceName(queryValues.Get("noblanks"))
	var noblanks bool
	if noblanksStr == "true" {
		noblanks = true
	}
	var formatStr string
	if len(parts) >= 8 {
		formatStr = parts[7]
	}

	// Construct the index for this tile
	plane := dvid.DataShapeString(planeStr)
	shape, err := plane.DataShape()
	if err != nil {
		err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	scaling, err := strconv.ParseUint(scalingStr, 10, 8)
	if err != nil {
		err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	tileCoord, err := dvid.StringToPoint(coordStr, "_")
	if err != nil {
		err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	if tileCoord.NumDims() != 3 {
		err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr)
		server.BadRequest(w, r, err)
		return err
	}
	indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)}
	data, err := d.getTileData(ctx, shape, Scaling(scaling), indexZYX)
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}
	if data == nil {
		if noblanks {
			http.NotFound(w, r)
			return nil
		}
		img, err := d.getBlankTileImage(uuid, shape, Scaling(scaling))
		if err != nil {
			return err
		}
		return dvid.WriteImageHttp(w, img, formatStr)
	}

	switch d.Encoding {
	case LZ4:
		var img dvid.Image
		if err := img.Deserialize(data); err != nil {
			return err
		}
		data, err = img.GetPNG()
		w.Header().Set("Content-type", "image/png")
	case PNG:
		w.Header().Set("Content-type", "image/png")
	case JPG:
		w.Header().Set("Content-type", "image/jpeg")
	}
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}
	if _, err = w.Write(data); err != nil {
		return err
	}
	return nil
}
Example #9
0
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
	// TODO -- Refactor this method to break it up and make it simpler.  Use the web routing for the endpoints.

	timedLog := dvid.NewTimeLog()

	// Get the action (GET, POST)
	action := strings.ToLower(r.Method)
	switch action {
	case "get":
	case "post":
	case "delete":
	default:
		server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
		return
	}

	// Break URL request into arguments
	url := r.URL.Path[len(server.WebAPIPath):]
	parts := strings.Split(url, "/")
	if len(parts[len(parts)-1]) == 0 {
		parts = parts[:len(parts)-1]
	}

	// Get query strings and possible roi
	var roiptr *imageblk.ROI
	queryValues := r.URL.Query()
	roiname := dvid.InstanceName(queryValues.Get("roi"))
	if len(roiname) != 0 {
		roiptr = new(imageblk.ROI)
	}

	// Handle POST on data -> setting of configuration
	if len(parts) == 3 && action == "post" {
		config, err := server.DecodeJSON(r)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := d.ModifyConfig(config); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := datastore.SaveDataByUUID(uuid, d); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
		return
	}

	if len(parts) < 4 {
		server.BadRequest(w, r, "Incomplete API request")
		return
	}

	// Process help and info.
	switch parts[3] {
	case "help":
		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprintln(w, dtype.Help())

	case "metadata":
		jsonStr, err := d.NdDataMetadata()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
		fmt.Fprintln(w, jsonStr)

	case "info":
		jsonBytes, err := d.MarshalJSON()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, string(jsonBytes))

	case "label":
		// GET <api URL>/node/<UUID>/<data name>/label/<coord>
		if len(parts) < 5 {
			server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
			return
		}
		coord, err := dvid.StringToPoint(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-type", "application/json")
		jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
		fmt.Fprintf(w, jsonStr)
		timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)

	case "labels":
		// POST <api URL>/node/<UUID>/<data name>/labels
		if action != "post" {
			server.BadRequest(w, r, "Batch labels query must be a POST request")
			return
		}
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			server.BadRequest(w, r, "Bad POSTed data for batch reverse query.  Should be JSON.")
			return
		}
		var coords []dvid.Point3d
		if err := json.Unmarshal(data, &coords); err != nil {
			server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
			return
		}
		w.Header().Set("Content-type", "application/json")
		fmt.Fprintf(w, "[")
		sep := false
		for _, coord := range coords {
			label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if sep {
				fmt.Fprintf(w, ",")
			}
			fmt.Fprintf(w, "%d", label)
			sep = true
		}
		fmt.Fprintf(w, "]")
		timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))

	case "blocks":
		// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
		if len(parts) < 6 {
			server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
			return
		}
		blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		span, err := strconv.Atoi(parts[5])
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if action != "delete" {
			server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
			return
		}
		if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)

	case "pseudocolor":
		if len(parts) < 7 {
			server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
			return
		}
		shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
		planeStr := dvid.DataShapeString(shapeStr)
		plane, err := planeStr.DataShape()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		switch plane.ShapeDimensions() {
		case 2:
			slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action != "get" {
				server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
				return
			}
			lbl, err := d.NewLabels(slice, nil)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if roiptr != nil {
				roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}

			// Convert to pseudocolor
			pseudoColor, err := colorImage(img)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}

			//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
			var formatStr string
			if len(parts) >= 8 {
				formatStr = parts[7]
			}
			err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
		default:
			server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
			return
		}

	case "raw", "isotropic":
		if len(parts) < 7 {
			server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
			return
		}
		var isotropic bool = (parts[3] == "isotropic")
		shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
		planeStr := dvid.DataShapeString(shapeStr)
		plane, err := planeStr.DataShape()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		switch plane.ShapeDimensions() {
		case 2:
			slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action != "get" {
				server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
				return
			}
			rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
			lbl, err := d.NewLabels(rawSlice, nil)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if roiptr != nil {
				roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if isotropic {
				dstW := int(slice.Size().Value(0))
				dstH := int(slice.Size().Value(1))
				img, err = img.ScaleImage(dstW, dstH)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			var formatStr string
			if len(parts) >= 8 {
				formatStr = parts[7]
			}
			//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
			err = dvid.WriteImageHttp(w, img.Get(), formatStr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
		case 3:
			queryStrings := r.URL.Query()
			throttle := queryStrings.Get("throttle")
			if throttle == "true" || throttle == "on" {
				select {
				case <-server.Throttle:
					// Proceed with operation, returning throttle token to server at end.
					defer func() {
						server.Throttle <- 1
					}()
				default:
					throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
						server.MaxThrottledOps)
					http.Error(w, throttleMsg, http.StatusServiceUnavailable)
					return
				}
			}
			compression := queryValues.Get("compression")
			subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action == "get" {
				lbl, err := d.NewLabels(subvol, nil)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				if roiptr != nil {
					roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				}
				data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				w.Header().Set("Content-type", "application/octet-stream")
				switch compression {
				case "":
					_, err = w.Write(data)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				case "lz4":
					compressed := make([]byte, lz4.CompressBound(data))
					var n, outSize int
					if outSize, err = lz4.Compress(data, compressed); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					compressed = compressed[:outSize]
					if n, err = w.Write(compressed); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if n != outSize {
						errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
						dvid.Errorf(errmsg)
						server.BadRequest(w, r, errmsg)
						return
					}
				case "gzip":
					gw := gzip.NewWriter(w)
					if _, err = gw.Write(data); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if err = gw.Close(); err != nil {
						server.BadRequest(w, r, err)
						return
					}
				default:
					server.BadRequest(w, r, "unknown compression type %q", compression)
					return
				}
			} else {
				if isotropic {
					server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
					return
				}
				// Make sure vox is block-aligned
				if !dvid.BlockAligned(subvol, d.BlockSize()) {
					server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
					return
				}

				var data []byte
				switch compression {
				case "":
					tlog := dvid.NewTimeLog()
					data, err = ioutil.ReadAll(r.Body)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					tlog.Debugf("read 3d uncompressed POST")
				case "lz4":
					tlog := dvid.NewTimeLog()
					data, err = ioutil.ReadAll(r.Body)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if len(data) == 0 {
						server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
						return
					}
					tlog.Debugf("read 3d lz4 POST")
					tlog = dvid.NewTimeLog()
					uncompressed := make([]byte, subvol.NumVoxels()*8)
					err = lz4.Uncompress(data, uncompressed)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					data = uncompressed
					tlog.Debugf("uncompressed 3d lz4 POST")
				case "gzip":
					tlog := dvid.NewTimeLog()
					gr, err := gzip.NewReader(r.Body)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					data, err = ioutil.ReadAll(gr)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if err = gr.Close(); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					tlog.Debugf("read and uncompress 3d gzip POST")
				default:
					server.BadRequest(w, r, "unknown compression type %q", compression)
					return
				}
				lbl, err := d.NewLabels(subvol, data)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				if roiptr != nil {
					roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				}
				if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
		default:
			server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
			return
		}

	default:
		server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q.  See API help.",
			parts[3], d.DataName())
	}
}
Example #10
0
// PutLocal adds image data to a version node, altering underlying blocks if the image
// intersects the block.
//
// The image filename glob MUST BE absolute file paths that are visible to the server.
// This function is meant for mass ingestion of large data files, and it is inappropriate
// to read gigabytes of data just to send it over the network to a local DVID.
func (d *Data) PutLocal(request datastore.Request, reply *datastore.Response) error {
	timedLog := dvid.NewTimeLog()

	// Parse the request
	var uuidStr, dataName, cmdStr, sourceStr, planeStr, offsetStr string
	filenames := request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr,
		&planeStr, &offsetStr)
	if len(filenames) == 0 {
		return fmt.Errorf("Need to include at least one file to add: %s", request)
	}

	// Get offset
	offset, err := dvid.StringToPoint(offsetStr, ",")
	if err != nil {
		return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
	}

	// Get list of files to add
	var addedFiles string
	if len(filenames) == 1 {
		addedFiles = filenames[0]
	} else {
		addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
	}
	dvid.Debugf(addedFiles + "\n")

	// Get plane
	plane, err := dvid.DataShapeString(planeStr).DataShape()
	if err != nil {
		return err
	}

	// Get Repo and IDs
	uuid, versionID, err := datastore.MatchingUUID(uuidStr)
	if err != nil {
		return err
	}

	// Load and PUT each image.
	numSuccessful := 0
	for _, filename := range filenames {
		sliceLog := dvid.NewTimeLog()
		img, _, err := dvid.GoImageFromFile(filename)
		if err != nil {
			return fmt.Errorf("Error after %d images successfully added: %v", numSuccessful, err)
		}
		slice, err := dvid.NewOrthogSlice(plane, offset, dvid.RectSize(img.Bounds()))
		if err != nil {
			return fmt.Errorf("Unable to determine slice: %v", err)
		}

		vox, err := d.NewVoxels(slice, img)
		if err != nil {
			return err
		}
		storage.FileBytesRead <- len(vox.Data())
		if err = d.PutVoxels(versionID, vox, nil); err != nil {
			return err
		}
		sliceLog.Debugf("%s put local %s", d.DataName(), slice)
		numSuccessful++
		offset = offset.Add(dvid.Point3d{0, 0, 1})
	}

	if err := datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
		return err
	}
	timedLog.Infof("RPC put local (%s) completed", addedFiles)
	return nil
}
Example #11
0
// DoRPC acts as a switchboard for RPC commands.
func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error {
	switch req.TypeCommand() {
	case "load":
		if len(req.Command) < 5 {
			return fmt.Errorf("Poorly formatted load command.  See command-line help.")
		}
		// Parse the request
		var uuidStr, dataName, cmdStr, offsetStr string
		filenames, err := req.FilenameArgs(1, &uuidStr, &dataName, &cmdStr, &offsetStr)
		if err != nil {
			return err
		}
		if len(filenames) == 0 {
			hostname, _ := os.Hostname()
			return fmt.Errorf("Couldn't find any files to add.  Are they visible to DVID server on %s?",
				hostname)
		}

		// Get offset
		offset, err := dvid.StringToPoint(offsetStr, ",")
		if err != nil {
			return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
		}

		// Get list of files to add
		var addedFiles string
		if len(filenames) == 1 {
			addedFiles = filenames[0]
		} else {
			addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
		}
		dvid.Debugf(addedFiles + "\n")

		uuid, versionID, err := datastore.MatchingUUID(uuidStr)
		if err != nil {
			return err
		}
		if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil {
			return err
		}
		return d.LoadImages(versionID, offset, filenames)

	case "put":
		if len(req.Command) < 7 {
			return fmt.Errorf("Poorly formatted put command.  See command-line help.")
		}
		source := req.Command[4]
		switch source {
		case "local":
			return d.PutLocal(req, reply)
		case "remote":
			return fmt.Errorf("put remote not yet implemented")
		default:
			return fmt.Errorf("Unknown command.  Data instance '%s' [%s] does not support '%s' command.",
				d.DataName(), d.TypeName(), req.TypeCommand())
		}

	case "roi":
		if len(req.Command) < 6 {
			return fmt.Errorf("Poorly formatted roi command. See command-line help.")
		}
		return d.ForegroundROI(req, reply)

	default:
		return fmt.Errorf("Unknown command.  Data instance '%s' [%s] does not support '%s' command.",
			d.DataName(), d.TypeName(), req.TypeCommand())
	}
	return nil
}
Example #12
0
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
	timedLog := dvid.NewTimeLog()
	versionID := ctx.VersionID()

	// Get the action (GET, POST)
	action := strings.ToLower(r.Method)

	// Break URL request into arguments
	url := r.URL.Path[len(server.WebAPIPath):]
	parts := strings.Split(url, "/")
	if len(parts[len(parts)-1]) == 0 {
		parts = parts[:len(parts)-1]
	}

	// Handle POST on data -> setting of configuration
	if len(parts) == 3 && action == "put" {
		config, err := server.DecodeJSON(r)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := d.ModifyConfig(config); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := datastore.SaveDataByUUID(uuid, d); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
		return
	}

	if len(parts) < 4 {
		server.BadRequest(w, r, "Incomplete API request")
		return
	}

	// Process help and info.
	switch parts[3] {
	case "help":
		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprintln(w, dtype.Help())

	case "info":
		jsonBytes, err := d.MarshalJSON()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, string(jsonBytes))

	case "sparsevol":
		// GET <api URL>/node/<UUID>/<data name>/sparsevol/<label>
		// POST <api URL>/node/<UUID>/<data name>/sparsevol/<label>
		if len(parts) < 5 {
			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol' command")
			return
		}
		label, err := strconv.ParseUint(parts[4], 10, 64)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		switch action {
		case "get":
			queryValues := r.URL.Query()
			var b Bounds
			b.VoxelBounds, err = dvid.BoundsFromQueryString(r)
			if err != nil {
				server.BadRequest(w, r, "Error parsing bounds from query string: %v\n", err)
				return
			}
			b.BlockBounds = b.VoxelBounds.Divide(d.BlockSize)
			b.Exact = queryValues.Get("exact") == "true"
			data, err := GetSparseVol(ctx, label, b)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			w.Header().Set("Content-type", "application/octet-stream")
			_, err = w.Write(data)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
		case "post":
			server.BadRequest(w, r, "POST of sparsevol not currently implemented\n")
			return
			// if err := d.PutSparseVol(versionID, label, r.Body); err != nil {
			// 	server.BadRequest(w, r, err)
			// 	return
			// }
		default:
			server.BadRequest(w, r, "Unable to handle HTTP action %s on sparsevol endpoint", action)
			return
		}
		timedLog.Infof("HTTP %s: sparsevol on label %d (%s)", r.Method, label, r.URL)

	case "sparsevol-by-point":
		// GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord>
		if len(parts) < 5 {
			server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'sparsevol-by-point' command")
			return
		}
		coord, err := dvid.StringToPoint(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		source, err := d.GetSyncedLabelblk(versionID)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		label, err := source.GetLabelAtPoint(ctx.VersionID(), coord)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		data, err := GetSparseVol(ctx, label, Bounds{})
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-type", "application/octet-stream")
		_, err = w.Write(data)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: sparsevol-by-point at %s (%s)", r.Method, coord, r.URL)

	case "sparsevol-coarse":
		// GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label>
		if len(parts) < 5 {
			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol-coarse' command")
			return
		}
		label, err := strconv.ParseUint(parts[4], 10, 64)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		data, err := GetSparseCoarseVol(ctx, label)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-type", "application/octet-stream")
		_, err = w.Write(data)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: sparsevol-coarse on label %d (%s)", r.Method, label, r.URL)

	case "maxlabel":
		// GET <api URL>/node/<UUID>/<data name>/maxlabel
		w.Header().Set("Content-Type", "application/json")
		switch action {
		case "get":
			maxlabel, ok := d.MaxLabel[versionID]
			if !ok {
				server.BadRequest(w, r, "No maximum label found for %s version %d\n", d.DataName(), versionID)
				return
			}
			fmt.Fprintf(w, "{%q: %d}", "maxlabel", maxlabel)
		default:
			server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
			return
		}
		timedLog.Infof("HTTP maxlabel request (%s)", r.URL)

	case "nextlabel":
		// GET <api URL>/node/<UUID>/<data name>/nextlabel
		// POST <api URL>/node/<UUID>/<data name>/nextlabel
		w.Header().Set("Content-Type", "application/json")
		switch action {
		case "get":
			fmt.Fprintf(w, "{%q: %d}", "nextlabel", d.MaxRepoLabel+1)
		case "post":
			server.BadRequest(w, r, "POST on maxlabel is not supported yet.\n")
			return
		default:
			server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
			return
		}
		timedLog.Infof("HTTP maxlabel request (%s)", r.URL)

	case "split":
		// POST <api URL>/node/<UUID>/<data name>/split/<label>
		if action != "post" {
			server.BadRequest(w, r, "Split requests must be POST actions.")
			return
		}
		if len(parts) < 5 {
			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command")
			return
		}
		fromLabel, err := strconv.ParseUint(parts[4], 10, 64)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		toLabel, err := d.SplitLabels(ctx.VersionID(), fromLabel, r.Body)
		if err != nil {
			server.BadRequest(w, r, fmt.Sprintf("Error on split: %v", err))
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, "{%q: %d}", "label", toLabel)
		timedLog.Infof("HTTP split request (%s)", r.URL)

	case "merge":
		// POST <api URL>/node/<UUID>/<data name>/merge
		if action != "post" {
			server.BadRequest(w, r, "Merge requests must be POST actions.")
			return
		}
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			server.BadRequest(w, r, "Bad POSTed data for merge.  Should be JSON.")
			return
		}
		var tuple labels.MergeTuple
		if err := json.Unmarshal(data, &tuple); err != nil {
			server.BadRequest(w, r, fmt.Sprintf("Bad merge op JSON: %v", err))
			return
		}
		mergeOp, err := tuple.Op()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := d.MergeLabels(ctx.VersionID(), mergeOp); err != nil {
			server.BadRequest(w, r, fmt.Sprintf("Error on merge: %v", err))
			return
		}
		timedLog.Infof("HTTP merge request (%s)", r.URL)

	default:
		server.BadRequest(w, r, "Unrecognized API call %q for labelvol data %q.  See API help.",
			parts[3], d.DataName())
	}
}