// instanceSelector retrieves the data instance given its complete string name and // forwards the request to that instance's HTTP handler. func instanceSelector(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { var err error dataname := dvid.InstanceName(c.URLParams["dataname"]) uuid, ok := c.Env["uuid"].(dvid.UUID) if !ok { msg := fmt.Sprintf("Bad format for UUID %q\n", c.Env["uuid"]) BadRequest(w, r, msg) return } data, err := datastore.GetDataByUUID(uuid, dataname) if err != nil { BadRequest(w, r, err) return } v, err := datastore.VersionFromUUID(uuid) if err != nil { BadRequest(w, r, err) } ctx := datastore.NewVersionedCtx(data, v) // Handle DVID-wide query string commands like non-interactive call designations queryValues := r.URL.Query() // All HTTP requests are interactive so let server tally request. interactive := queryValues.Get("interactive") if interactive == "" || (interactive != "false" && interactive != "0") { GotInteractiveRequest() } // TODO: setup routing for data instances as well. data.ServeHTTP(uuid, ctx, w, r) } return http.HandlerFunc(fn) }
func repoDeleteInstanceHandler(c web.C, w http.ResponseWriter, r *http.Request) { uuid := c.Env["uuid"].(dvid.UUID) queryValues := r.URL.Query() imsure := queryValues.Get("imsure") if imsure != "true" { BadRequest(w, r, "Cannot delete instance unless query string 'imsure=true' is present!") return } dataname, ok := c.URLParams["dataname"] if !ok { BadRequest(w, r, "Error retrieving data instance name from URL parameters") return } // Make sure this instance exists. _, err := datastore.GetDataByUUID(uuid, dvid.InstanceName(dataname)) if err != nil { BadRequest(w, r, "Error trying to delete %q for UUID %s: %v", dataname, uuid, err) return } // Do the deletion. Under hood, modifies metadata immediately and launches async k/v deletion. if err := datastore.DeleteDataByUUID(uuid, dvid.InstanceName(dataname)); err != nil { BadRequest(w, r, "Error deleting data instance %q: %v", dataname, err) return } // Just respond that deletion was successfully started w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, `{"result": "Started deletion of data instance %q from repo with root %s"}`, dataname, uuid) }
// getBlankTileData returns zero 2d tile data with a given scaling and format. func (d *Data) getBlankTileImage(uuid dvid.UUID, shape dvid.DataShape, scaling Scaling) (image.Image, error) { levelSpec, found := d.Levels[scaling] if !found { return nil, fmt.Errorf("Could not extract tiles for unspecified scale level %d", scaling) } tileW, tileH, err := shape.GetSize2D(levelSpec.TileSize) if err != nil { return nil, err } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { return nil, err } src, ok := source.(*imageblk.Data) if !ok { return nil, fmt.Errorf("Data instance %q for uuid %q is not imageblk.Data", d.Source, uuid) } bytesPerVoxel := src.Values.BytesPerElement() switch bytesPerVoxel { case 1, 2, 4, 8: numBytes := tileW * tileH * bytesPerVoxel data := make([]byte, numBytes, numBytes) return dvid.GoImageFromData(data, int(tileW), int(tileH)) default: return nil, fmt.Errorf("Cannot construct blank tile for data %q with %d bytes/voxel", d.Source, src.Values.BytesPerElement()) } }
func TestDeleteInstance(t *testing.T) { tests.UseStore() defer tests.CloseStore() apiStr := fmt.Sprintf("%srepos", server.WebAPIPath) r := server.TestHTTP(t, "POST", apiStr, nil) var jsonResp map[string]interface{} if err := json.Unmarshal(r, &jsonResp); err != nil { t.Fatalf("Unable to unmarshal repo creation response: %s\n", string(r)) } v, ok := jsonResp["root"] if !ok { t.Fatalf("No 'root' metadata returned: %s\n", string(r)) } uuidStr, ok := v.(string) if !ok { t.Fatalf("Couldn't cast returned 'root' data (%v) into string.\n", v) } uuid := dvid.UUID(uuidStr) // Add a data instance. var config dvid.Config server.CreateTestInstance(t, uuid, "keyvalue", "foo", config) // Make sure it exists. _, err := datastore.GetDataByUUID(uuid, "foo") if err != nil { t.Errorf("Couldn't create data instance 'foo'\n") } // Shouldn't be able to delete instance without "imsure" delReq := fmt.Sprintf("%srepo/%s/%s", server.WebAPIPath, uuid, "foo") server.TestBadHTTP(t, "DELETE", delReq, nil) delReq = fmt.Sprintf("%srepo/%s/%s?imsure=true", server.WebAPIPath, uuid, "foo") server.TestHTTP(t, "DELETE", delReq, nil) // Make sure it no longer exists. _, err = datastore.GetDataByUUID(uuid, "foo") if err == nil { t.Errorf("Shouldn't be able to access a deleted data instance 'foo'\n") } }
// GetByUUID returns a pointer to labelsurf data given a version (UUID) and data name. func GetByUUID(uuid dvid.UUID, name dvid.InstanceName) (*Data, error) { source, err := datastore.GetDataByUUID(uuid, name) if err != nil { return nil, err } data, ok := source.(*Data) if !ok { return nil, fmt.Errorf("Instance '%s' is not a labelsurf datatype!", name) } return data, nil }
func TestMultiscale2dRepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() // Make source uuid, _ := initTestRepo() makeGrayscale(uuid, t, "grayscale") // Make labels and set various properties config := dvid.NewConfig() config.Set("Placeholder", "true") config.Set("Format", "jpg") config.Set("Source", "grayscale") dataservice, err := datastore.NewData(uuid, mstype, "myimagetile", config) if err != nil { t.Errorf("Unable to create imagetile instance: %v\n", err) } msdata, ok := dataservice.(*Data) if !ok { t.Fatalf("Can't cast imagetile data service into imagetile.Data\n") } oldData := *msdata // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, msdata); err != nil { t.Fatalf("Unable to save repo during imagetile persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice2, err := datastore.GetDataByUUID(uuid, "myimagetile") if err != nil { t.Fatalf("Can't get keyvalue instance from reloaded test db: %v\n", err) } msdata2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not imagetile.Data\n") } if !reflect.DeepEqual(oldData.Properties, msdata2.Properties) { t.Errorf("Expected properties %v, got %v\n", oldData.Properties, msdata2.Properties) } }
func TestLabelblkRepoPersistence(t *testing.T) { tests.UseStore() defer tests.CloseStore() uuid, _ := initTestRepo() // Make labels and set various properties config := dvid.NewConfig() config.Set("BlockSize", "12,13,14") config.Set("VoxelSize", "1.1,2.8,11") config.Set("VoxelUnits", "microns,millimeters,nanometers") dataservice, err := datastore.NewData(uuid, labelsT, "mylabels", config) if err != nil { t.Errorf("Unable to create labels instance: %v\n", err) } labels, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast labels data service into Data\n") } oldData := *labels // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, labels); err != nil { t.Fatalf("Unable to save repo during labels persistence test: %v\n", err) } tests.CloseReopenStore() dataservice2, err := datastore.GetDataByUUID(uuid, "mylabels") if err != nil { t.Fatalf("Can't get labels instance from reloaded test db: %v\n", err) } labels2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not imageblk.Data\n") } if !oldData.Equals(labels2) { t.Errorf("Expected %v, got %v\n", oldData, *labels2) } }
func TestGrayscaleRepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() uuid, _ := initTestRepo() // Make grayscale and set various properties config := dvid.NewConfig() config.Set("BlockSize", "12,13,14") config.Set("VoxelSize", "1.1,2.8,11") config.Set("VoxelUnits", "microns,millimeters,nanometers") dataservice, err := datastore.NewData(uuid, grayscaleT, "mygrayscale", config) if err != nil { t.Errorf("Unable to create grayscale instance: %s\n", err) } grayscale, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast uint8 data service into Data\n") } oldData := *grayscale // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, grayscale); err != nil { t.Fatalf("Unable to save repo during grayscale persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice2, err := datastore.GetDataByUUID(uuid, "mygrayscale") if err != nil { t.Fatalf("Can't get grayscale instance from reloaded test db: %v\n", err) } grayscale2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not imageblk.Data\n") } if !oldData.Equals(grayscale2) { t.Errorf("Expected %v, got %v\n", oldData, *grayscale2) } }
func TestMultichan16RepoPersistence(t *testing.T) { tests.UseStore() defer tests.CloseStore() uuid, _ := initTestRepo() // Make labels and set various properties config := dvid.NewConfig() dataservice, err := datastore.NewData(uuid, dtype, "mymultichan16", config) if err != nil { t.Errorf("Unable to create multichan16 instance: %v\n", err) } mcdata, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast multichan16 data service into multichan16.Data\n") } oldData := *mcdata // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, mcdata); err != nil { t.Fatalf("Unable to save repo during multichan16 persistence test: %v\n", err) } tests.CloseReopenStore() dataservice2, err := datastore.GetDataByUUID(uuid, "mymultichan16") if err != nil { t.Fatalf("Can't get multichan16 instance from reloaded test db: %v\n", err) } mcdata2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not multichan16.Data\n") } if !oldData.Equals(mcdata2) { t.Errorf("Expected %v, got %v\n", oldData, *mcdata2) } }
func TestKeyvalueRepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() uuid, _ := initTestRepo() // Make labels and set various properties config := dvid.NewConfig() dataservice, err := datastore.NewData(uuid, kvtype, "mykv", config) if err != nil { t.Errorf("Unable to create keyvalue instance: %v\n", err) } kvdata, ok := dataservice.(*Data) if !ok { t.Errorf("Can't cast keyvalue data service into keyvalue.Data\n") } oldData := *kvdata // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, kvdata); err != nil { t.Fatalf("Unable to save repo during keyvalue persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice2, err := datastore.GetDataByUUID(uuid, "mykv") if err != nil { t.Fatalf("Can't get keyvalue instance from reloaded test db: %v\n", err) } kvdata2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not keyvalue.Data\n") } if !oldData.Equals(kvdata2) { t.Errorf("Expected %v, got %v\n", oldData, *kvdata2) } }
// ServeHTTP handles all incoming HTTP requests for this data. func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) { timedLog := dvid.NewTimeLog() action := strings.ToLower(r.Method) switch action { case "get", "post": // Acceptable default: server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs") return } // Break URL request into arguments url := r.URL.Path[len(server.WebAPIPath):] parts := strings.Split(url, "/") if len(parts[len(parts)-1]) == 0 { parts = parts[:len(parts)-1] } if len(parts) < 4 { server.BadRequest(w, r, "incomplete API request") return } switch parts[3] { case "help": w.Header().Set("Content-Type", "text/plain") fmt.Fprintln(w, d.Help()) case "info": jsonBytes, err := d.MarshalJSON() if err != nil { server.BadRequest(w, r, err) return } w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, string(jsonBytes)) case "tile": if action == "post" { server.BadRequest(w, r, "DVID does not yet support POST of imagetile") return } if err := d.ServeTile(uuid, ctx, w, r, parts); err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: tile (%s)", r.Method, r.URL) case "raw", "isotropic": if action == "post" { server.BadRequest(w, r, "imagetile '%s' can only PUT tiles not images", d.DataName()) return } if len(parts) < 7 { server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3]) return } shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6] planeStr := dvid.DataShapeString(shapeStr) plane, err := planeStr.DataShape() if err != nil { server.BadRequest(w, r, err) return } if plane.ShapeDimensions() != 2 { server.BadRequest(w, r, "Quadtrees can only return 2d images not %s", plane) return } slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_") if err != nil { server.BadRequest(w, r, err) return } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { server.BadRequest(w, r, err) return } src, ok := source.(*imageblk.Data) if !ok { server.BadRequest(w, r, "Cannot construct imagetile for non-voxels data: %s", d.Source) return } img, err := d.GetImage(ctx, src, slice, parts[3] == "isotropic") if err != nil { server.BadRequest(w, r, err) return } var formatStr string if len(parts) >= 8 { formatStr = parts[7] } err = dvid.WriteImageHttp(w, img.Get(), formatStr) if err != nil { server.BadRequest(w, r, err) return } timedLog.Infof("HTTP %s: tile-accelerated %s %s (%s)", r.Method, planeStr, parts[3], r.URL) default: server.BadRequest(w, r, "Illegal request for imagetile data. See 'help' for REST API") } }
// Returns the default tile spec that will fully cover the source extents and scaling 0 // uses the original voxel resolutions with each subsequent scale causing a 2x zoom out. func (d *Data) DefaultTileSpec(uuidStr string) (TileSpec, error) { uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return nil, err } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { return nil, err } var ok bool var src *imageblk.Data src, ok = source.(*imageblk.Data) if !ok { return nil, fmt.Errorf("Cannot construct tile spec for non-voxels data: %s", d.Source) } // Set scaling 0 based on extents and resolution of source. //extents := src.Extents() resolution := src.Properties.Resolution.VoxelSize if len(resolution) != 3 { return nil, fmt.Errorf("Cannot construct tile spec for non-3d data: voxel is %d-d", len(resolution)) } // Expand min and max points to coincide with full tile boundaries of highest resolution. minTileCoord := src.MinPoint.(dvid.Chunkable).Chunk(DefaultTileSize) maxTileCoord := src.MaxPoint.(dvid.Chunkable).Chunk(DefaultTileSize) minTiledPt := minTileCoord.MinPoint(DefaultTileSize) maxTiledPt := maxTileCoord.MaxPoint(DefaultTileSize) sizeVolume := maxTiledPt.Sub(minTiledPt).AddScalar(1) dvid.Infof("Creating default multiscale tile spec for volume of size %s\n", sizeVolume) // For each dimension, calculate the number of scaling levels necessary to cover extent, // assuming we use the raw resolution at scaling 0. numScales := make([]int, 3) var maxScales int var dim uint8 for dim = 0; dim < 3; dim++ { numPixels := float64(sizeVolume.Value(dim)) tileSize := float64(DefaultTileSize.Value(dim)) if numPixels <= tileSize { numScales[dim] = 1 } else { numScales[dim] = int(math.Ceil(math.Log2(numPixels/tileSize))) + 1 } if numScales[dim] > maxScales { maxScales = numScales[dim] } } // Initialize the tile level specification specs := make(TileSpec, maxScales) curRes := resolution levelMag := dvid.Point3d{2, 2, 2} var scaling Scaling for scaling = 0; scaling < Scaling(maxScales); scaling++ { for dim = 0; dim < 3; dim++ { if scaling >= Scaling(numScales[dim]) { levelMag[dim] = 1 } } specs[scaling] = TileScaleSpec{ LevelSpec{curRes, DefaultTileSize}, levelMag, } curRes = curRes.MultScalar(2.0) } return specs, nil }
func (d *Data) ConstructTiles(uuidStr string, tileSpec TileSpec, request datastore.Request) error { config := request.Settings() uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { return err } src, ok := source.(*imageblk.Data) if !ok { return fmt.Errorf("Cannot construct imagetile for non-voxels data: %s", d.Source) } // Save the current tile specification d.Levels = tileSpec if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } // Get size of tile at lowest resolution. lastLevel := Scaling(len(tileSpec) - 1) loresSpec, found := tileSpec[lastLevel] if !found { return fmt.Errorf("Illegal tile spec. Should have levels 0 to absent %d.", lastLevel) } var loresSize [3]float64 for i := 0; i < 3; i++ { loresSize[i] = float64(loresSpec.Resolution[i]) * float64(DefaultTileSize[i]) } loresMag := dvid.Point3d{1, 1, 1} for i := Scaling(0); i < lastLevel; i++ { levelMag := tileSpec[i].levelMag loresMag[0] *= levelMag[0] loresMag[1] *= levelMag[1] loresMag[2] *= levelMag[2] } // Get min and max points in terms of distance. var minPtDist, maxPtDist [3]float64 for i := uint8(0); i < 3; i++ { minPtDist[i] = float64(src.MinPoint.Value(i)) * float64(src.VoxelSize[i]) maxPtDist[i] = float64(src.MaxPoint.Value(i)) * float64(src.VoxelSize[i]) } // Adjust min and max points for the tileable surface at lowest resolution. var minTiledPt, maxTiledPt dvid.Point3d for i := 0; i < 3; i++ { minInt, _ := math.Modf(minPtDist[i] / loresSize[i]) maxInt, _ := math.Modf(maxPtDist[i] / loresSize[i]) minTileCoord := int32(minInt) maxTileCoord := int32(maxInt) minTiledPt[i] = minTileCoord * DefaultTileSize[i] * loresMag[i] maxTiledPt[i] = (maxTileCoord+1)*DefaultTileSize[i]*loresMag[i] - 1 } sizeVolume := maxTiledPt.Sub(minTiledPt).AddScalar(1) // Setup swappable ExtData buffers (the stitched slices) so we can be generating tiles // at same time we are reading and stitching them. var bufferLock [2]sync.Mutex var sliceBuffers [2]*imageblk.Voxels var bufferNum int // Get the planes we should tile. planes, err := config.GetShapes("planes", ";") if planes == nil { // If no planes are specified, construct imagetile for 3 orthogonal planes. planes = []dvid.DataShape{dvid.XY, dvid.XZ, dvid.YZ} } outF, err := d.putTileFunc(versionID) // sort the tile spec keys to iterate from highest to lowest resolution var sortedKeys []int for scaling, _ := range tileSpec { sortedKeys = append(sortedKeys, int(scaling)) } sort.Ints(sortedKeys) for _, plane := range planes { timedLog := dvid.NewTimeLog() offset := minTiledPt.Duplicate() switch { case plane.Equals(dvid.XY): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling XY image %d x %d pixels\n", width, height) for z := src.MinPoint.Value(2); z <= src.MaxPoint.Value(2); z++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xy]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{2: z}) slice, err := dvid.NewOrthogSlice(dvid.XY, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled XY Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Infof("Read XY Tile @ Z = %d, now tiling...", z) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate XY Tiles") case plane.Equals(dvid.XZ): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling XZ image %d x %d pixels\n", width, height) for y := src.MinPoint.Value(1); y <= src.MaxPoint.Value(1); y++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xz]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{1: y}) slice, err := dvid.NewOrthogSlice(dvid.XZ, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled XZ Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Infof("Read XZ Tile @ Y = %d, now tiling...", y) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate XZ Tiles") case plane.Equals(dvid.YZ): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling YZ image %d x %d pixels\n", width, height) for x := src.MinPoint.Value(0); x <= src.MaxPoint.Value(0); x++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [yz]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{0: x}) slice, err := dvid.NewOrthogSlice(dvid.YZ, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] outF, err := d.putTileFunc(versionID) if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled YZ Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Debugf("Read YZ Tile @ X = %d, now tiling...", x) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate YZ Tiles") default: dvid.Infof("Skipping request to tile '%s'. Unsupported.", plane) } } return nil }
func repoResolveHandler(c web.C, w http.ResponseWriter, r *http.Request) { uuid, _, err := datastore.MatchingUUID(c.URLParams["uuid"]) if err != nil { BadRequest(w, r, err) return } if r.Body == nil { BadRequest(w, r, "merge resolving requires JSON to be POSTed per API documentation") return } data, err := ioutil.ReadAll(r.Body) if err != nil { BadRequest(w, r, err) return } jsonData := struct { Data []dvid.InstanceName `json:"data"` Note string `json:"note"` Parents []string `json:"parents"` }{} if err := json.Unmarshal(data, &jsonData); err != nil { BadRequest(w, r, fmt.Sprintf("Malformed JSON request in body: %v", err)) return } if len(jsonData.Data) == 0 { BadRequest(w, r, "Must specify at least one data instance using 'data' field") return } if len(jsonData.Parents) < 2 { BadRequest(w, r, "Must specify at least two parent UUIDs using 'parents' field") return } // Convert JSON of parents into []UUID and whether we need a child for them to add deletions. numParents := len(jsonData.Parents) oldParents := make([]dvid.UUID, numParents) newParents := make([]dvid.UUID, numParents, numParents) // UUID of any parent extension for deletions. for i, uuidFrag := range jsonData.Parents { uuid, _, err := datastore.MatchingUUID(uuidFrag) if err != nil { BadRequest(w, r, fmt.Sprintf("can't match parent %q: %v", uuidFrag, err)) return } oldParents[i] = uuid newParents[i] = dvid.NilUUID } // Iterate through all k/v for given data instances, making sure we find any conflicts. // If any are found, remove them with first UUIDs taking priority. for _, name := range jsonData.Data { data, err := datastore.GetDataByUUID(uuid, name) if err != nil { BadRequest(w, r, err) return } if err := datastore.DeleteConflicts(uuid, data, oldParents, newParents); err != nil { BadRequest(w, r, fmt.Errorf("Conflict deletion error for data %q: %v", data.DataName(), err)) return } } // If we have any new nodes to accomodate deletions, commit them. for i, oldUUID := range oldParents { if newParents[i] != oldUUID { err := datastore.Commit(newParents[i], "Version for deleting conflicts before merge", nil) if err != nil { BadRequest(w, r, "Error while creating new nodes to handle required deletions: %v", err) return } } } // Do the merge mt := datastore.MergeConflictFree newuuid, err := datastore.Merge(newParents, jsonData.Note, mt) if err != nil { BadRequest(w, r, err) } else { w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, "{%q: %q}", "child", newuuid) } }
// Do acts as a switchboard for remote command execution func (c *RPCConnection) Do(cmd datastore.Request, reply *datastore.Response) error { if reply == nil { dvid.Debugf("reply is nil coming in!\n") return nil } if cmd.Name() == "" { return fmt.Errorf("Server error: got empty command!") } switch cmd.Name() { case "help": reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress()) case "shutdown": Shutdown() // Make this process shutdown in a second to allow time for RPC to finish. // TODO -- Better way to do this? log.Printf("DVID server halted due to 'shutdown' command.") reply.Text = fmt.Sprintf("DVID server at %s has been halted.\n", config.RPCAddress()) go func() { time.Sleep(1 * time.Second) os.Exit(0) }() case "types": if len(cmd.Command) == 1 { text := "\nData Types within this DVID Server\n" text += "----------------------------------\n" mapTypes, err := datastore.Types() if err != nil { return fmt.Errorf("Error trying to retrieve data types within this DVID server!") } for url, typeservice := range mapTypes { text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url) } reply.Text = text } else { if len(cmd.Command) != 3 || cmd.Command[2] != "help" { return fmt.Errorf("Unknown types command: %q", cmd.Command) } var typename string cmd.CommandArgs(1, &typename) typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename)) if err != nil { return err } reply.Text = typeservice.Help() } case "repos": var subcommand, alias, description, uuidStr string cmd.CommandArgs(1, &subcommand, &alias, &description, &uuidStr) switch subcommand { case "new": var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } root, err := datastore.NewRepo(alias, description, assign) if err != nil { return err } if err := datastore.SetRepoAlias(root, alias); err != nil { return err } if err := datastore.SetRepoDescription(root, description); err != nil { return err } reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root) default: return fmt.Errorf("Unknown repos command: %q", subcommand) } case "repo": var uuidStr, subcommand string cmd.CommandArgs(1, &uuidStr, &subcommand) uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } switch subcommand { case "new": var typename, dataname string cmd.CommandArgs(3, &typename, &dataname) // Get TypeService typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename)) if err != nil { return err } // Create new data config := cmd.Settings() _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config) if err != nil { return err } reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "branch": cmd.CommandArgs(3, &uuidStr) var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } child, err := datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign) if err != nil { return err } reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "merge": uuids := cmd.CommandArgs(2) parents := make([]dvid.UUID, len(uuids)+1) parents[0] = dvid.UUID(uuid) i := 1 for uuid := range uuids { parents[i] = dvid.UUID(uuid) i++ } child, err := datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree) if err != nil { return err } reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "push": /* var target string cmd.CommandArgs(3, &target) config := cmd.Settings() if err = datastore.Push(repo, target, config); err != nil { return err } reply.Text = fmt.Sprintf("Repo %q pushed to %q\n", repo.RootUUID(), target) */ return fmt.Errorf("push command has been temporarily suspended") default: return fmt.Errorf("Unknown command: %q", cmd) } case "node": var uuidStr, descriptor string cmd.CommandArgs(1, &uuidStr, &descriptor) uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Get the DataService dataname := dvid.InstanceName(descriptor) var subcommand string cmd.CommandArgs(3, &subcommand) dataservice, err := datastore.GetDataByUUID(uuid, dataname) if err != nil { return err } if subcommand == "help" { reply.Text = dataservice.Help() return nil } return dataservice.DoRPC(cmd, reply) default: return fmt.Errorf("Unknown command: '%s'", cmd) } return nil }
func TestROIRepoPersistence(t *testing.T) { datastore.OpenTest() defer datastore.CloseTest() uuid, _ := initTestRepo() // Add data config := dvid.NewConfig() dataservice1, err := datastore.NewData(uuid, roitype, "myroi", config) if err != nil { t.Errorf("Error creating new roi instance: %v\n", err) } roi1, ok := dataservice1.(*Data) if !ok { t.Errorf("Returned new data instance 1 is not roi.Data\n") } if roi1.DataName() != "myroi" { t.Errorf("New roi data instance name set incorrectly: %q != %q\n", roi1.DataName(), "myroi") } config.Set("BlockSize", "15,16,17") dataservice2, err := datastore.NewData(uuid, roitype, "myroi2", config) if err != nil { t.Errorf("Error creating new roi instance: %v\n", err) } roi2, ok := dataservice2.(*Data) if !ok { t.Errorf("Returned new data instance 2 is not roi.Data\n") } roi2.MinZ = 13 roi2.MaxZ = 3098 oldData := *roi2 // Check instance IDs if roi1.InstanceID() == roi2.InstanceID() { t.Errorf("Instance IDs should be different: %d == %d\n", roi1.InstanceID(), roi2.InstanceID()) } // Restart test datastore and see if datasets are still there. if err = datastore.SaveDataByUUID(uuid, dataservice1); err != nil { t.Fatalf("Unable to save data1 during ROI persistence test: %v\n", err) } if err = datastore.SaveDataByUUID(uuid, dataservice2); err != nil { t.Fatalf("Unable to save data2 during ROI persistence test: %v\n", err) } datastore.CloseReopenTest() dataservice3, err := datastore.GetDataByUUID(uuid, "myroi2") if err != nil { t.Fatalf("Can't get first ROI instance from reloaded test db: %v\n", err) } roi2new, ok := dataservice3.(*Data) if !ok { t.Errorf("Returned new data instance 3 is not roi.Data\n") } if !oldData.Equals(roi2new) { t.Errorf("Expected %v, got %v\n", oldData, *roi2new) } }
// CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, grayscaleName, destName string request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName) // Get the version uuid, v, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Log request if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } // Get the grayscale data. dataservice, err := datastore.GetDataByUUID(uuid, dvid.InstanceName(grayscaleName)) if err != nil { return err } grayscale, ok := dataservice.(*imageblk.Data) if !ok { return fmt.Errorf("%s is not the name of uint8 data", grayscaleName) } // Create a new rgba8blk data. var compservice datastore.DataService compservice, err = datastore.GetDataByUUID(uuid, dvid.InstanceName(destName)) if err == nil { return fmt.Errorf("Data instance with name %q already exists", destName) } typeService, err := datastore.TypeServiceByName("rgba8blk") if err != nil { return fmt.Errorf("Could not get rgba8 type service from DVID") } config := dvid.NewConfig() compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config) if err != nil { return err } composite, ok := compservice.(*imageblk.Data) if !ok { return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName) } // Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time. wg := new(sync.WaitGroup) op := &blockOp{grayscale, composite, v} chunkOp := &storage.ChunkOp{op, wg} store, err := storage.BigDataStore() if err != nil { return err } ctx := datastore.NewVersionedCtx(d, v) extents := d.Extents() blockBeg := imageblk.NewTKey(extents.MinIndex) blockEnd := imageblk.NewTKey(extents.MaxIndex) err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk)) wg.Wait() // Set new mapped data to same extents. composite.Properties.Extents = grayscale.Properties.Extents if err := datastore.SaveDataByUUID(uuid, composite); err != nil { dvid.Infof("Could not save new data '%s': %v\n", destName, err) } timedLog.Infof("Created composite of %s and %s", grayscaleName, destName) return nil }