// DoRPC acts as a switchboard for RPC commands. func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error { switch req.TypeCommand() { case "load": if len(req.Command) < 5 { return fmt.Errorf("Poorly formatted load command. See command-line help.") } // Parse the request var uuidStr, dataName, cmdStr, offsetStr string filenames, err := req.FilenameArgs(1, &uuidStr, &dataName, &cmdStr, &offsetStr) if err != nil { return err } if len(filenames) == 0 { return fmt.Errorf("Need to include at least one file to add: %s", req) } // Get offset offset, err := dvid.StringToPoint(offsetStr, ",") if err != nil { return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err) } // Get list of files to add var addedFiles string if len(filenames) == 1 { addedFiles = filenames[0] } else { addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1) } dvid.Debugf(addedFiles + "\n") uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil { return err } if err = d.LoadImages(versionID, offset, filenames); err != nil { return err } if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } return nil case "composite": if len(req.Command) < 6 { return fmt.Errorf("Poorly formatted composite command. See command-line help.") } return d.CreateComposite(req, reply) default: return fmt.Errorf("Unknown command. Data type '%s' [%s] does not support '%s' command.", d.DataName(), d.TypeName(), req.TypeCommand()) } return nil }
// SetMetadata loads JSON data giving MinTileCoord, MaxTileCoord, and tile level specifications. func (d *Data) SetMetadata(uuid dvid.UUID, jsonBytes []byte) error { var config metadataJSON if err := json.Unmarshal(jsonBytes, &config); err != nil { return err } tileSpec, err := parseTileSpec(config.Levels) if err != nil { return err } d.Levels = tileSpec d.MinTileCoord = config.MinTileCoord d.MaxTileCoord = config.MaxTileCoord if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } return nil }
func TestMultiscale2dRepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() // Make source uuid, _ := initTestRepo() makeGrayscale(uuid, t, "grayscale") // Make labels and set various properties config := dvid.NewConfig() config.Set("Placeholder", "true") config.Set("Format", "jpg") config.Set("Source", "grayscale") dataservice, err := datastore.NewData(uuid, mstype, "myimagetile", config) if err != nil { t.Errorf("Unable to create imagetile instance: %v\n", err) } msdata, ok := dataservice.(*Data) if !ok { t.Fatalf("Can't cast imagetile data service into imagetile.Data\n") } oldData := *msdata // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, msdata); err != nil { t.Fatalf("Unable to save repo during imagetile persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice2, err := datastore.GetDataByUUID(uuid, "myimagetile") if err != nil { t.Fatalf("Can't get keyvalue instance from reloaded test db: %v\n", err) } msdata2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not imagetile.Data\n") } if !reflect.DeepEqual(oldData.Properties, msdata2.Properties) { t.Errorf("Expected properties %v, got %v\n", oldData.Properties, msdata2.Properties) } }
func TestFloat32RepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() uuid, _ := initTestRepo() // Make grayscale and set various properties config := dvid.NewConfig() config.Set("BlockSize", "12,13,14") config.Set("VoxelSize", "1.1,2.8,11") config.Set("VoxelUnits", "microns,millimeters,nanometers") dataservice, err := datastore.NewData(uuid, floatimgT, "floatimg", config) if err != nil { t.Errorf("Unable to create float32 instance: %s\n", err) } floatimg, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast float32 data service into Data\n") } oldData := *floatimg // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, floatimg); err != nil { t.Fatalf("Unable to save repo during floatimg persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice2, err := datastore.GetDataByUUIDName(uuid, "floatimg") if err != nil { t.Fatalf("Can't get floatimg instance from reloaded test db: %v\n", err) } floatimg2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not imageblk.Data\n") } if !oldData.Equals(floatimg2) { t.Errorf("Expected %v, got %v\n", oldData, *floatimg2) } }
func TestLabelblkRepoPersistence(t *testing.T) { tests.UseStore() defer tests.CloseStore() uuid, _ := initTestRepo() // Make labels and set various properties config := dvid.NewConfig() config.Set("BlockSize", "12,13,14") config.Set("VoxelSize", "1.1,2.8,11") config.Set("VoxelUnits", "microns,millimeters,nanometers") dataservice, err := datastore.NewData(uuid, labelsT, "mylabels", config) if err != nil { t.Errorf("Unable to create labels instance: %v\n", err) } labels, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast labels data service into Data\n") } oldData := *labels // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, labels); err != nil { t.Fatalf("Unable to save repo during labels persistence test: %v\n", err) } tests.CloseReopenStore() dataservice2, err := datastore.GetDataByUUID(uuid, "mylabels") if err != nil { t.Fatalf("Can't get labels instance from reloaded test db: %v\n", err) } labels2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not imageblk.Data\n") } if !oldData.Equals(labels2) { t.Errorf("Expected %v, got %v\n", oldData, *labels2) } }
func TestKeyvalueRepoPersistence(t *testing.T) { tests.UseStore() defer tests.CloseStore() uuid, _ := initTestRepo() // Make labels and set various properties config := dvid.NewConfig() dataservice, err := datastore.NewData(uuid, kvtype, "mykv", config) if err != nil { t.Errorf("Unable to create keyvalue instance: %v\n", err) } kvdata, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast keyvalue data service into keyvalue.Data\n") } oldData := *kvdata // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, kvdata); err != nil { t.Fatalf("Unable to save repo during keyvalue persistence test: %v\n", err) } tests.CloseReopenStore() dataservice2, err := datastore.GetDataByUUID(uuid, "mykv") if err != nil { t.Fatalf("Can't get keyvalue instance from reloaded test db: %v\n", err) } kvdata2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not keyvalue.Data\n") } if !oldData.Equals(kvdata2) { t.Errorf("Expected %v, got %v\n", oldData, *kvdata2) } }
func TestMultichan16RepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() uuid, _ := initTestRepo() // Make labels and set various properties config := dvid.NewConfig() dataservice, err := datastore.NewData(uuid, dtype, "mymultichan16", config) if err != nil { t.Errorf("Unable to create multichan16 instance: %v\n", err) } mcdata, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast multichan16 data service into multichan16.Data\n") } oldData := *mcdata // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, mcdata); err != nil { t.Fatalf("Unable to save repo during multichan16 persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice2, err := datastore.GetDataByUUIDName(uuid, "mymultichan16") if err != nil { t.Fatalf("Can't get multichan16 instance from reloaded test db: %v\n", err) } mcdata2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not multichan16.Data\n") } if !oldData.Equals(mcdata2) { t.Errorf("Expected %v, got %v\n", oldData, *mcdata2) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() // versionID := ctx.VersionID() // Get the action (GET, POST) action := strings.ToLower(r.Method) // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "put" { config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, dtype.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "sync": if action != "post" { server.BadRequest(w, r, "Only POST allowed to sync endpoint") return } replace := r.URL.Query().Get("replace") == "true" if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil { server.BadRequest(w, r, err) return } case "label": if action != "get" { server.BadRequest(w, r, "Only GET action is available on 'label' endpoint.") return } if len(parts) < 5 { server.BadRequest(w, r, "Must include label after 'label' endpoint.") return } label, err := strconv.ParseUint(parts[4], 10, 64) if err != nil { server.BadRequest(w, r, err) return } if label == 0 { server.BadRequest(w, r, "Label 0 is protected background value and cannot be used for query.") return } queryStrings := r.URL.Query() jsonBytes, err := d.GetLabelJSON(ctx, label, queryStrings.Get("relationships") == "true") if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") if _, err := w.Write(jsonBytes); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: get synaptic elements for label %d (%s)", r.Method, label, r.URL) case "tag": if action != "get" { server.BadRequest(w, r, "Only GET action is available on 'tag' endpoint.") return } if len(parts) < 5 { server.BadRequest(w, r, "Must include tag string after 'tag' endpoint.") return } tag := Tag(parts[4]) queryStrings := r.URL.Query() jsonBytes, err := d.GetTagJSON(ctx, tag, queryStrings.Get("relationships") == "true") if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") if _, err := w.Write(jsonBytes); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: get synaptic elements for tag %s (%s)", r.Method, tag, r.URL) case "elements": switch action { case "get": // GET <api URL>/node/<UUID>/<data name>/elements/<size>/<offset> if len(parts) < 6 { server.BadRequest(w, r, "Expect size and offset to follow 'elements' in GET request") return } sizeStr, offsetStr := parts[4], parts[5] ext3d, err := dvid.NewExtents3dFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } elements, err := d.GetRegionSynapses(ctx, ext3d) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") jsonBytes, err := json.Marshal(elements) if err != nil { server.BadRequest(w, r, err) return } if _, err := w.Write(jsonBytes); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: synapse elements in subvolume (size %s, offset %s) (%s)", r.Method, sizeStr, offsetStr, r.URL) case "post": if err := d.StoreSynapses(ctx, r.Body); err != nil { server.BadRequest(w, r, err) return } default: server.BadRequest(w, r, "Only GET or POST action is available on 'elements' endpoint.") return } case "element": // DELETE <api URL>/node/<UUID>/<data name>/element/<coord> if action != "delete" { server.BadRequest(w, r, "Only DELETE action is available on 'element' endpoint.") return } if len(parts) < 5 { server.BadRequest(w, r, "Must include coordinate after DELETE on 'element' endpoint.") return } pt, err := dvid.StringToPoint3d(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } if err := d.DeleteElement(ctx, pt); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: delete synaptic element at %s (%s)", r.Method, pt, r.URL) case "move": // POST <api URL>/node/<UUID>/<data name>/move/<from_coord>/<to_coord> if action != "post" { server.BadRequest(w, r, "Only POST action is available on 'move' endpoint.") return } if len(parts) < 6 { server.BadRequest(w, r, "Must include 'from' and 'to' coordinate after 'move' endpoint.") return } fromPt, err := dvid.StringToPoint3d(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } toPt, err := dvid.StringToPoint3d(parts[5], "_") if err != nil { server.BadRequest(w, r, err) return } if err := d.MoveElement(ctx, fromPt, toPt); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: move synaptic element from %s to %s (%s)", r.Method, fromPt, toPt, r.URL) default: server.BadAPIRequest(w, r, d) } }
// LoadLocal adds image data to a version node. See HelpMessage for example of // command-line use of "load local". func (d *Data) LoadLocal(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, sourceStr, filename string _ = request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr, &filename) // Get the uuid from a uniquely identifiable string uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return fmt.Errorf("Could not find node with UUID %s: %v", uuidStr, err) } // Load the V3D Raw file. ext := filepath.Ext(filename) switch ext { case ".raw", ".v3draw": default: return fmt.Errorf("Unknown extension '%s' when expected V3D Raw file", ext) } file, err := os.Open(filename) if err != nil { return err } unmarshaler := V3DRawMarshaler{} channels, err := unmarshaler.UnmarshalV3DRaw(file) if err != nil { return err } // Store the metadata d.NumChannels = len(channels) d.Properties.Values = make(dvid.DataValues, d.NumChannels) if d.NumChannels > 0 { reply.Text = fmt.Sprintf("Loaded %s into data '%s': found %d channels\n", d.DataName(), filename, d.NumChannels) reply.Text += fmt.Sprintf(" %s", channels[0]) } else { reply.Text = fmt.Sprintf("Found no channels in file %s\n", filename) return nil } for i, channel := range channels { d.Properties.Values[i] = channel.Voxels.Values()[0] } // Get repo and save it. if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } // PUT each channel of the file into the datastore using a separate data name. for _, channel := range channels { dvid.Infof("Processing channel %d... \n", channel.channelNum) err = d.PutVoxels(versionID, channel.Voxels, nil) if err != nil { return err } } // Create a RGB composite from the first 3 channels. This is considered to be channel 0 // or can be accessed with the base data name. dvid.Infof("Creating composite image from channels...\n") err = d.storeComposite(versionID, channels) if err != nil { return err } timedLog.Infof("RPC load local '%s' completed", filename) return nil }
func (d *Data) ConstructTiles(uuidStr string, tileSpec TileSpec, request datastore.Request) error { config := request.Settings() uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { return err } src, ok := source.(*imageblk.Data) if !ok { return fmt.Errorf("Cannot construct imagetile for non-voxels data: %s", d.Source) } // Save the current tile specification d.Levels = tileSpec if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } // Get size of tile at lowest resolution. lastLevel := Scaling(len(tileSpec) - 1) loresSpec, found := tileSpec[lastLevel] if !found { return fmt.Errorf("Illegal tile spec. Should have levels 0 to absent %d.", lastLevel) } var loresSize [3]float64 for i := 0; i < 3; i++ { loresSize[i] = float64(loresSpec.Resolution[i]) * float64(DefaultTileSize[i]) } loresMag := dvid.Point3d{1, 1, 1} for i := Scaling(0); i < lastLevel; i++ { levelMag := tileSpec[i].levelMag loresMag[0] *= levelMag[0] loresMag[1] *= levelMag[1] loresMag[2] *= levelMag[2] } // Get min and max points in terms of distance. var minPtDist, maxPtDist [3]float64 for i := uint8(0); i < 3; i++ { minPtDist[i] = float64(src.MinPoint.Value(i)) * float64(src.VoxelSize[i]) maxPtDist[i] = float64(src.MaxPoint.Value(i)) * float64(src.VoxelSize[i]) } // Adjust min and max points for the tileable surface at lowest resolution. var minTiledPt, maxTiledPt dvid.Point3d for i := 0; i < 3; i++ { minInt, _ := math.Modf(minPtDist[i] / loresSize[i]) maxInt, _ := math.Modf(maxPtDist[i] / loresSize[i]) minTileCoord := int32(minInt) maxTileCoord := int32(maxInt) minTiledPt[i] = minTileCoord * DefaultTileSize[i] * loresMag[i] maxTiledPt[i] = (maxTileCoord+1)*DefaultTileSize[i]*loresMag[i] - 1 } sizeVolume := maxTiledPt.Sub(minTiledPt).AddScalar(1) // Setup swappable ExtData buffers (the stitched slices) so we can be generating tiles // at same time we are reading and stitching them. var bufferLock [2]sync.Mutex var sliceBuffers [2]*imageblk.Voxels var bufferNum int // Get the planes we should tile. planes, err := config.GetShapes("planes", ";") if planes == nil { // If no planes are specified, construct imagetile for 3 orthogonal planes. planes = []dvid.DataShape{dvid.XY, dvid.XZ, dvid.YZ} } outF, err := d.putTileFunc(versionID) // sort the tile spec keys to iterate from highest to lowest resolution var sortedKeys []int for scaling, _ := range tileSpec { sortedKeys = append(sortedKeys, int(scaling)) } sort.Ints(sortedKeys) for _, plane := range planes { timedLog := dvid.NewTimeLog() offset := minTiledPt.Duplicate() switch { case plane.Equals(dvid.XY): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling XY image %d x %d pixels\n", width, height) for z := src.MinPoint.Value(2); z <= src.MaxPoint.Value(2); z++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xy]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{2: z}) slice, err := dvid.NewOrthogSlice(dvid.XY, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled XY Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Infof("Read XY Tile @ Z = %d, now tiling...", z) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate XY Tiles") case plane.Equals(dvid.XZ): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling XZ image %d x %d pixels\n", width, height) for y := src.MinPoint.Value(1); y <= src.MaxPoint.Value(1); y++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xz]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{1: y}) slice, err := dvid.NewOrthogSlice(dvid.XZ, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled XZ Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Infof("Read XZ Tile @ Y = %d, now tiling...", y) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate XZ Tiles") case plane.Equals(dvid.YZ): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling YZ image %d x %d pixels\n", width, height) for x := src.MinPoint.Value(0); x <= src.MaxPoint.Value(0); x++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [yz]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{0: x}) slice, err := dvid.NewOrthogSlice(dvid.YZ, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] outF, err := d.putTileFunc(versionID) if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled YZ Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Debugf("Read YZ Tile @ X = %d, now tiling...", x) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate YZ Tiles") default: dvid.Infof("Skipping request to tile '%s'. Unsupported.", plane) } } return nil }
// CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, grayscaleName, destName string request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName) // Get the version uuid, v, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Log request if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } // Get the grayscale data. dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(grayscaleName)) if err != nil { return err } grayscale, ok := dataservice.(*imageblk.Data) if !ok { return fmt.Errorf("%s is not the name of uint8 data", grayscaleName) } // Create a new rgba8blk data. var compservice datastore.DataService compservice, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(destName)) if err == nil { return fmt.Errorf("Data instance with name %q already exists", destName) } typeService, err := datastore.TypeServiceByName("rgba8blk") if err != nil { return fmt.Errorf("Could not get rgba8 type service from DVID") } config := dvid.NewConfig() compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config) if err != nil { return err } composite, ok := compservice.(*imageblk.Data) if !ok { return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName) } // Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time. wg := new(sync.WaitGroup) op := &compositeOp{grayscale, composite, v} chunkOp := &storage.ChunkOp{op, wg} store, err := d.GetOrderedKeyValueDB() if err != nil { return err } ctx := datastore.NewVersionedCtx(d, v) extents := d.Extents() blockBeg := imageblk.NewTKey(extents.MinIndex) blockEnd := imageblk.NewTKey(extents.MaxIndex) err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk)) wg.Wait() // Set new mapped data to same extents. composite.Properties.Extents = grayscale.Properties.Extents if err := datastore.SaveDataByUUID(uuid, composite); err != nil { dvid.Infof("Could not save new data '%s': %v\n", destName, err) } timedLog.Infof("Created composite of %s and %s", grayscaleName, destName) return nil }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { // TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints. timedLog := dvid.NewTimeLog() // Get the action (GET, POST) action := strings.ToLower(r.Method) switch action { case "get": case "post": case "delete": default: server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Get query strings and possible roi var roiptr *imageblk.ROI queryValues := r.URL.Query() roiname := dvid.InstanceName(queryValues.Get("roi")) if len(roiname) != 0 { roiptr = new(imageblk.ROI) } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "post" { config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, dtype.Help()) case "metadata": jsonStr, err := d.NdDataMetadata() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json") fmt.Fprintln(w, jsonStr) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "label": // GET <api URL>/node/<UUID>/<data name>/label/<coord> if len(parts) < 5 { server.BadRequest(w, r, "DVID requires coord to follow 'label' command") return } coord, err := dvid.StringToPoint(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } label, err := d.GetLabelAtPoint(ctx.VersionID(), coord) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") jsonStr := fmt.Sprintf(`{"Label": %d}`, label) fmt.Fprintf(w, jsonStr) timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL) case "labels": // POST <api URL>/node/<UUID>/<data name>/labels if action != "post" { server.BadRequest(w, r, "Batch labels query must be a POST request") return } data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.") return } var coords []dvid.Point3d if err := json.Unmarshal(data, &coords); err != nil { server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err)) return } w.Header().Set("Content-type", "application/json") fmt.Fprintf(w, "[") sep := false for _, coord := range coords { label, err := d.GetLabelAtPoint(ctx.VersionID(), coord) if err != nil { server.BadRequest(w, r, err) return } if sep { fmt.Fprintf(w, ",") } fmt.Fprintf(w, "%d", label) sep = true } fmt.Fprintf(w, "]") timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords)) case "blocks": // DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX> if len(parts) < 6 { server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.") return } blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } span, err := strconv.Atoi(parts[5]) if err != nil { server.BadRequest(w, r, err) return } if action != "delete" { server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint") return } if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL) case "pseudocolor": if len(parts) < 7 { server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3]) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 2: slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action != "get" { server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores") return } lbl, err := d.NewLabels(slice, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } img, err := d.GetImage(ctx.VersionID(), lbl, roiptr) if err != nil { server.BadRequest(w, r, err) return } // Convert to pseudocolor pseudoColor, err := colorImage(img) if err != nil { server.BadRequest(w, r, err) return } //dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice) var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, pseudoColor, formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL) default: server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests") return } case "raw", "isotropic": if len(parts) < 7 { server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3]) return } var isotropic bool = (parts[3] == "isotropic") shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 2: slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action != "get" { server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores") return } rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic) lbl, err := d.NewLabels(rawSlice, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } img, err := d.GetImage(ctx.VersionID(), lbl, roiptr) if err != nil { server.BadRequest(w, r, err) return } if isotropic { dstW := int(slice.Size().Value(0)) dstH := int(slice.Size().Value(1)) img, err = img.ScaleImage(dstW, dstH) if err != nil { server.BadRequest(w, r, err) return } } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } //dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice) err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL) case 3: queryStrings := r.URL.Query() throttle := queryStrings.Get("throttle") if throttle == "true" || throttle == "on" { select { case <-server.Throttle: // Proceed with operation, returning throttle token to server at end. defer func() { server.Throttle <- 1 }() default: throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations", server.MaxThrottledOps) http.Error(w, throttleMsg, http.StatusServiceUnavailable) return } } compression := queryValues.Get("compression") subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action == "get" { lbl, err := d.NewLabels(subvol, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") switch compression { case "": _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } case "lz4": compressed := make([]byte, lz4.CompressBound(data)) var n, outSize int if outSize, err = lz4.Compress(data, compressed); err != nil { server.BadRequest(w, r, err) return } compressed = compressed[:outSize] if n, err = w.Write(compressed); err != nil { server.BadRequest(w, r, err) return } if n != outSize { errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize) dvid.Errorf(errmsg) server.BadRequest(w, r, errmsg) return } case "gzip": gw := gzip.NewWriter(w) if _, err = gw.Write(data); err != nil { server.BadRequest(w, r, err) return } if err = gw.Close(); err != nil { server.BadRequest(w, r, err) return } default: server.BadRequest(w, r, "unknown compression type %q", compression) return } } else { if isotropic { server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images") return } // Make sure vox is block-aligned if !dvid.BlockAligned(subvol, d.BlockSize()) { server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint()) return } var data []byte switch compression { case "": tlog := dvid.NewTimeLog() data, err = ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } tlog.Debugf("read 3d uncompressed POST") case "lz4": tlog := dvid.NewTimeLog() data, err = ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } if len(data) == 0 { server.BadRequest(w, r, "received 0 LZ4 compressed bytes") return } tlog.Debugf("read 3d lz4 POST") tlog = dvid.NewTimeLog() uncompressed := make([]byte, subvol.NumVoxels()*8) err = lz4.Uncompress(data, uncompressed) if err != nil { server.BadRequest(w, r, err) return } data = uncompressed tlog.Debugf("uncompressed 3d lz4 POST") case "gzip": tlog := dvid.NewTimeLog() gr, err := gzip.NewReader(r.Body) if err != nil { server.BadRequest(w, r, err) return } data, err = ioutil.ReadAll(gr) if err != nil { server.BadRequest(w, r, err) return } if err = gr.Close(); err != nil { server.BadRequest(w, r, err) return } tlog.Debugf("read and uncompress 3d gzip POST") default: server.BadRequest(w, r, "unknown compression type %q", compression) return } lbl, err := d.NewLabels(subvol, data) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl) if err != nil { server.BadRequest(w, r, err) return } } if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL) default: server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions") return } default: server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.", parts[3], d.DataName()) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() // Get the action (GET, POST) action := strings.ToLower(r.Method) // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "put" { config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, dtype.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "sync": if action != "post" { server.BadRequest(w, r, "Only POST allowed to sync endpoint") return } replace := r.URL.Query().Get("replace") == "true" if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil { server.BadRequest(w, r, err) return } case "count": if action != "get" { server.BadRequest(w, r, "Only GET action is available on 'count' endpoint.") return } if len(parts) < 6 { server.BadRequest(w, r, "Must include label and element type after 'count' endpoint.") return } label, err := strconv.ParseUint(parts[4], 10, 64) if err != nil { server.BadRequest(w, r, err) return } i := StringToIndexType(parts[5]) if i == UnknownIndex { server.BadRequest(w, r, fmt.Errorf("unknown index type specified (%q)", parts[5])) return } count, err := d.GetCountElementType(ctx, label, i) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") jsonStr := fmt.Sprintf(`{"Label":%d,%q:%d}`, label, i, count) if _, err := io.WriteString(w, jsonStr); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: get count for label %d, index type %s: %s", r.Method, label, i, r.URL) case "top": if action != "get" { server.BadRequest(w, r, "Only GET action is available on 'top' endpoint.") return } if len(parts) < 6 { server.BadRequest(w, r, "Must include N and element type after 'top' endpoint.") return } n, err := strconv.ParseUint(parts[4], 10, 32) if err != nil { server.BadRequest(w, r, err) return } i := StringToIndexType(parts[5]) if i == UnknownIndex { server.BadRequest(w, r, fmt.Errorf("unknown index type specified (%q)", parts[5])) return } labelSizes, err := d.GetTopElementType(ctx, int(n), i) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") jsonBytes, err := json.Marshal(labelSizes) if err != nil { server.BadRequest(w, r, err) return } if _, err := w.Write(jsonBytes); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: get top %d labels for index type %s: %s", r.Method, n, i, r.URL) case "threshold": if action != "get" { server.BadRequest(w, r, "Only GET action is available on 'threshold' endpoint.") return } if len(parts) < 6 { server.BadRequest(w, r, "Must include threshold # and element type after 'threshold' endpoint.") return } t, err := strconv.ParseUint(parts[4], 10, 32) if err != nil { server.BadRequest(w, r, err) return } minSize := uint32(t) i := StringToIndexType(parts[5]) if i == UnknownIndex { server.BadRequest(w, r, fmt.Errorf("unknown index type specified (%q)", parts[5])) return } queryStrings := r.URL.Query() var num, offset int offsetStr := queryStrings.Get("offset") if offsetStr != "" { offset, err = strconv.Atoi(offsetStr) if err != nil { server.BadRequest(w, r, fmt.Errorf("bad offset specified in query string (%q)", offsetStr)) return } } numStr := queryStrings.Get("n") if numStr != "" { num, err = strconv.Atoi(numStr) if err != nil { server.BadRequest(w, r, fmt.Errorf("bad num specified in query string (%q)", numStr)) return } } labels, err := d.GetLabelsByThreshold(ctx, i, minSize, offset, num) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/json") jsonBytes, err := json.Marshal(labels) if err != nil { server.BadRequest(w, r, err) return } if _, err := w.Write(jsonBytes); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: get %d labels for index type %s with threshold %d: %s", r.Method, num, i, t, r.URL) default: server.BadAPIRequest(w, r, d) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() action := strings.ToLower(r.Method) switch action { case "get", "post": // Acceptable default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } if len(parts) < 4 { server.BadRequest(w, r, "incomplete API request") return } switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "metadata": switch action { case "post": jsonBytes, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } tilespec, err := LoadTileSpec(jsonBytes) if err != nil { server.BadRequest(w, r, err) return } d.Levels = tilespec if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } case "get": if d.Levels == nil || len(d.Levels) == 0 { server.BadRequest(w, r, "tile metadata for imagetile %q was not set\n", d.DataName()) return } jsonBytes, err := d.Levels.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) } timedLog.Infof("HTTP %s: metadata (%s)", r.Method, r.URL) case "tile": switch action { case "post": err := d.PostTile(uuid, ctx, w, r, parts) if err != nil { server.BadRequest(w, r, "Error in posting tile with URL %q: %v\n", url, err) return } case "get": if err := d.ServeTile(uuid, ctx, w, r, parts); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL) case "tilekey": switch action { case "get": var err error var hexkey string if hexkey, err = d.GetTileKey(uuid, ctx, w, r, parts); err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, `{"key": "%s"}`, hexkey) timedLog.Infof("HTTP %s: tilekey (%s) returns %s", r.Method, r.URL, hexkey) default: server.BadRequest(w, r, fmt.Errorf("Cannot use HTTP %s for tilekey endpoint", action)) return } case "raw", "isotropic": if action == "post" { server.BadRequest(w, r, "imagetile '%s' can only PUT tiles not images", d.DataName()) return } if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3]) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } if plane.ShapeDimensions() != 2 { server.BadRequest(w, r, "Quadtrees can only return 2d images not %s", plane) return } slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { server.BadRequest(w, r, err) return } src, ok := source.(*imageblk.Data) if !ok { server.BadRequest(w, r, "Cannot construct imagetile for non-voxels data: %s", d.Source) return } img, err := d.GetImage(ctx, src, slice, parts[3] == "isotropic") if err != nil { server.BadRequest(w, r, err) return } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: tile-accelerated %s %s (%s)", r.Method, planeStr, parts[3], r.URL) default: server.BadRequest(w, r, "Illegal request for imagetile data. See 'help' for REST API") } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() // Get the action (GET, POST) action := strings.ToLower(r.Method) switch action { case "get": case "post": default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Get query strings and possible roi var roiptr *ROI queryValues := r.URL.Query() roiname := dvid.InstanceName(queryValues.Get("roi")) if len(roiname) != 0 { roiptr = new(ROI) attenuationStr := queryValues.Get("attenuation") if len(attenuationStr) != 0 { attenuation, err := strconv.Atoi(attenuationStr) if err != nil { server.BadRequest(w, r, err) return } if attenuation < 1 || attenuation > 7 { server.BadRequest(w, r, "Attenuation should be from 1 to 7 (divides by 2^n)") return } roiptr.attenuation = uint8(attenuation) } } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "put" { fmt.Printf("Setting configuration of data '%s'\n", d.DataName()) config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) return case "metadata": jsonStr, err := d.NdDataMetadata() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json") fmt.Fprintln(w, jsonStr) return case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) return case "rawkey": // GET <api URL>/node/<UUID>/<data name>/rawkey?x=<block x>&y=<block y>&z=<block z> if len(parts) != 4 { server.BadRequest(w, r, "rawkey endpoint should be followed by query strings (x, y, and z) giving block coord") return } case "blocks": // GET <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX> // POST <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX> if len(parts) < 6 { server.BadRequest(w, r, "%q must be followed by block-coord/span-x", parts[3]) return } blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } span, err := strconv.Atoi(parts[5]) if err != nil { server.BadRequest(w, r, err) return } if action == "get" { data, err := d.GetBlocks(ctx.VersionID(), blockCoord, int32(span)) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } } else { if err := d.PutBlocks(ctx.VersionID(), blockCoord, span, r.Body); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: Blocks (%s)", r.Method, r.URL) case "arb": // GET <api URL>/node/<UUID>/<data name>/arb/<top left>/<top right>/<bottom left>/<res>[/<format>] if len(parts) < 8 { server.BadRequest(w, r, "%q must be followed by top-left/top-right/bottom-left/res", parts[3]) return } queryStrings := r.URL.Query() throttle := queryStrings.Get("throttle") if throttle == "true" || throttle == "on" { select { case <-server.Throttle: // Proceed with operation, returning throttle token to server at end. defer func() { server.Throttle <- 1 }() default: throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations", server.MaxThrottledOps) http.Error(w, throttleMsg, http.StatusServiceUnavailable) return } } img, err := d.GetArbitraryImage(ctx, parts[4], parts[5], parts[6], parts[7]) if err != nil { server.BadRequest(w, r, err) return } var formatStr string if len(parts) >= 9 { formatStr = parts[8] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: Arbitrary image (%s)", r.Method, r.URL) case "raw", "isotropic": // GET <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>] if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3]) return } var isotropic bool = (parts[3] == "isotropic") shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } switch plane.ShapeDimensions() { case 2: slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action != "get" { server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores") return } rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic) if err != nil { server.BadRequest(w, r, err) return } vox, err := d.NewVoxels(rawSlice, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox) if err != nil { server.BadRequest(w, r, err) return } } img, err := d.GetImage(ctx.VersionID(), vox, roiptr) if err != nil { server.BadRequest(w, r, err) return } if isotropic { dstW := int(slice.Size().Value(0)) dstH := int(slice.Size().Value(1)) img, err = img.ScaleImage(dstW, dstH) if err != nil { server.BadRequest(w, r, err) return } } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL) case 3: queryStrings := r.URL.Query() if queryStrings.Get("throttle") == "on" { select { case <-server.Throttle: // Proceed with operation, returning throttle token to server at end. defer func() { server.Throttle <- 1 }() default: throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations", server.MaxThrottledOps) http.Error(w, throttleMsg, http.StatusServiceUnavailable) return } } subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } if action == "get" { vox, err := d.NewVoxels(subvol, nil) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox) if err != nil { server.BadRequest(w, r, err) return } } data, err := d.GetVolume(ctx.VersionID(), vox, roiptr) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } } else { if isotropic { err := fmt.Errorf("can only PUT 'raw' not 'isotropic' images") server.BadRequest(w, r, err) return } // Make sure vox is block-aligned if !dvid.BlockAligned(subvol, d.BlockSize()) { server.BadRequest(w, r, "cannot store voxels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint()) return } data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, err) return } vox, err := d.NewVoxels(subvol, data) if err != nil { server.BadRequest(w, r, err) return } if roiptr != nil { roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox) if err != nil { server.BadRequest(w, r, err) return } } if err = d.PutVoxels(ctx.VersionID(), vox, roiptr); err != nil { server.BadRequest(w, r, err) return } } timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL) default: server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions") } default: server.BadRequest(w, r, "Unrecognized API call for voxels %q. See API help.", d.DataName()) } }
func TestROIRepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() uuid, _ := initTestRepo() // Add data config := dvid.NewConfig() dataservice1, err := datastore.NewData(uuid, roitype, "myroi", config) if err != nil { t.Errorf("Error creating new roi instance: %v\n", err) } roi1, ok := dataservice1.(*Data) if !ok { t.Errorf("Returned new data instance 1 is not roi.Data\n") } if roi1.DataName() != "myroi" { t.Errorf("New roi data instance name set incorrectly: %q != %q\n", roi1.DataName(), "myroi") } config.Set("BlockSize", "15,16,17") dataservice2, err := datastore.NewData(uuid, roitype, "myroi2", config) if err != nil { t.Errorf("Error creating new roi instance: %v\n", err) } roi2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not roi.Data\n") } roi2.MinZ = 13 roi2.MaxZ = 3098 oldData := *roi2 // Check instance IDs if roi1.InstanceID() == roi2.InstanceID() { t.Errorf("Instance IDs should be different: %d == %d\n", roi1.InstanceID(), roi2.InstanceID()) } // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, dataservice1); err != nil { t.Fatalf("Unable to save data1 during ROI persistence test: %v\n", err) } if err = datastore.SaveDataByUUID(uuid, dataservice2); err != nil { t.Fatalf("Unable to save data2 during ROI persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice3, err := datastore.GetDataByUUID(uuid, "myroi2") if err != nil { t.Fatalf("Can't get first ROI instance from reloaded test db: %v\n", err) } roi2new, ok := dataservice3.(*Data) if !ok { t.Errorf("Returned new data instance 3 is not roi.Data\n") } if !oldData.Equals(roi2new) { t.Errorf("Expected %v, got %v\n", oldData, *roi2new) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() versionID := ctx.VersionID() // Get the action (GET, POST) action := strings.ToLower(r.Method) // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } // Handle POST on data -> setting of configuration if len(parts) == 3 && action == "put" { config, err := server.DecodeJSON(r) if err != nil { server.BadRequest(w, r, err) return } if err := d.ModifyConfig(config); err != nil { server.BadRequest(w, r, err) return } if err := datastore.SaveDataByUUID(uuid, d); err != nil { server.BadRequest(w, r, err) return } fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) return } if len(parts) < 4 { server.BadRequest(w, r, "Incomplete API request") return } // Process help and info. switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, dtype.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "sparsevol": // GET <api URL>/node/<UUID>/<data name>/sparsevol/<label> // POST <api URL>/node/<UUID>/<data name>/sparsevol/<label> if len(parts) < 5 { server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol' command") return } label, err := strconv.ParseUint(parts[4], 10, 64) if err != nil { server.BadRequest(w, r, err) return } switch action { case "get": queryValues := r.URL.Query() var b Bounds b.VoxelBounds, err = dvid.BoundsFromQueryString(r) if err != nil { server.BadRequest(w, r, "Error parsing bounds from query string: %v\n", err) return } b.BlockBounds = b.VoxelBounds.Divide(d.BlockSize) b.Exact = queryValues.Get("exact") == "true" data, err := GetSparseVol(ctx, label, b) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } case "post": server.BadRequest(w, r, "POST of sparsevol not currently implemented\n") return // if err := d.PutSparseVol(versionID, label, r.Body); err != nil { // server.BadRequest(w, r, err) // return // } default: server.BadRequest(w, r, "Unable to handle HTTP action %s on sparsevol endpoint", action) return } timedLog.Infof("HTTP %s: sparsevol on label %d (%s)", r.Method, label, r.URL) case "sparsevol-by-point": // GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord> if len(parts) < 5 { server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'sparsevol-by-point' command") return } coord, err := dvid.StringToPoint(parts[4], "_") if err != nil { server.BadRequest(w, r, err) return } source, err := d.GetSyncedLabelblk(versionID) if err != nil { server.BadRequest(w, r, err) return } label, err := source.GetLabelAtPoint(ctx.VersionID(), coord) if err != nil { server.BadRequest(w, r, err) return } data, err := GetSparseVol(ctx, label, Bounds{}) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: sparsevol-by-point at %s (%s)", r.Method, coord, r.URL) case "sparsevol-coarse": // GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label> if len(parts) < 5 { server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol-coarse' command") return } label, err := strconv.ParseUint(parts[4], 10, 64) if err != nil { server.BadRequest(w, r, err) return } data, err := GetSparseCoarseVol(ctx, label) if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-type", "application/octet-stream") _, err = w.Write(data) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: sparsevol-coarse on label %d (%s)", r.Method, label, r.URL) case "maxlabel": // GET <api URL>/node/<UUID>/<data name>/maxlabel w.Header().Set("Content-Type", "application/json") switch action { case "get": maxlabel, ok := d.MaxLabel[versionID] if !ok { server.BadRequest(w, r, "No maximum label found for %s version %d\n", d.DataName(), versionID) return } fmt.Fprintf(w, "{%q: %d}", "maxlabel", maxlabel) default: server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL) return } timedLog.Infof("HTTP maxlabel request (%s)", r.URL) case "nextlabel": // GET <api URL>/node/<UUID>/<data name>/nextlabel // POST <api URL>/node/<UUID>/<data name>/nextlabel w.Header().Set("Content-Type", "application/json") switch action { case "get": fmt.Fprintf(w, "{%q: %d}", "nextlabel", d.MaxRepoLabel+1) case "post": server.BadRequest(w, r, "POST on maxlabel is not supported yet.\n") return default: server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL) return } timedLog.Infof("HTTP maxlabel request (%s)", r.URL) case "split": // POST <api URL>/node/<UUID>/<data name>/split/<label> if action != "post" { server.BadRequest(w, r, "Split requests must be POST actions.") return } if len(parts) < 5 { server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command") return } fromLabel, err := strconv.ParseUint(parts[4], 10, 64) if err != nil { server.BadRequest(w, r, err) return } toLabel, err := d.SplitLabels(ctx.VersionID(), fromLabel, r.Body) if err != nil { server.BadRequest(w, r, fmt.Sprintf("Error on split: %v", err)) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, "{%q: %d}", "label", toLabel) timedLog.Infof("HTTP split request (%s)", r.URL) case "merge": // POST <api URL>/node/<UUID>/<data name>/merge if action != "post" { server.BadRequest(w, r, "Merge requests must be POST actions.") return } data, err := ioutil.ReadAll(r.Body) if err != nil { server.BadRequest(w, r, "Bad POSTed data for merge. Should be JSON.") return } var tuple labels.MergeTuple if err := json.Unmarshal(data, &tuple); err != nil { server.BadRequest(w, r, fmt.Sprintf("Bad merge op JSON: %v", err)) return } mergeOp, err := tuple.Op() if err != nil { server.BadRequest(w, r, err) return } if err := d.MergeLabels(ctx.VersionID(), mergeOp); err != nil { server.BadRequest(w, r, fmt.Sprintf("Error on merge: %v", err)) return } timedLog.Infof("HTTP merge request (%s)", r.URL) default: server.BadRequest(w, r, "Unrecognized API call %q for labelvol data %q. See API help.", parts[3], d.DataName()) } }