// DoRPC handles the 'generate' command. func (d *Data) DoRPC(request datastore.Request, reply *datastore.Response) error { if request.TypeCommand() != "generate" { return fmt.Errorf("Unknown command. Data instance '%s' [%s] does not support '%s' command.", d.DataName(), d.TypeName(), request.TypeCommand()) } var uuidStr, dataName, cmdStr string request.CommandArgs(1, &uuidStr, &dataName, &cmdStr) // Get the imagetile generation configuration from a file or stdin. var err error var tileSpec TileSpec if request.Input != nil { tileSpec, err = LoadTileSpec(request.Input) if err != nil { return err } } else { config := request.Settings() filename, found, err := config.GetString("filename") if err != nil { return err } if found { configData, err := storage.DataFromFile(filename) if err != nil { return err } tileSpec, err = LoadTileSpec(configData) if err != nil { return err } dvid.Infof("Using tile spec file: %s\n", filename) } else { dvid.Infof("Using default tile generation method since no tile spec file was given...\n") tileSpec, err = d.DefaultTileSpec(uuidStr) if err != nil { return err } } } reply.Text = fmt.Sprintf("Tiling data instance %q @ node %s...\n", dataName, uuidStr) go func() { err := d.ConstructTiles(uuidStr, tileSpec, request) if err != nil { dvid.Errorf("Cannot construct tiles for data instance %q @ node %s: %v\n", dataName, uuidStr, err) } }() return nil }
func (d *Data) ConstructTiles(uuidStr string, tileSpec TileSpec, request datastore.Request) error { config := request.Settings() uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } source, err := datastore.GetDataByUUID(uuid, d.Source) if err != nil { return err } src, ok := source.(*imageblk.Data) if !ok { return fmt.Errorf("Cannot construct imagetile for non-voxels data: %s", d.Source) } // Save the current tile specification d.Levels = tileSpec if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } // Get size of tile at lowest resolution. lastLevel := Scaling(len(tileSpec) - 1) loresSpec, found := tileSpec[lastLevel] if !found { return fmt.Errorf("Illegal tile spec. Should have levels 0 to absent %d.", lastLevel) } var loresSize [3]float64 for i := 0; i < 3; i++ { loresSize[i] = float64(loresSpec.Resolution[i]) * float64(DefaultTileSize[i]) } loresMag := dvid.Point3d{1, 1, 1} for i := Scaling(0); i < lastLevel; i++ { levelMag := tileSpec[i].levelMag loresMag[0] *= levelMag[0] loresMag[1] *= levelMag[1] loresMag[2] *= levelMag[2] } // Get min and max points in terms of distance. var minPtDist, maxPtDist [3]float64 for i := uint8(0); i < 3; i++ { minPtDist[i] = float64(src.MinPoint.Value(i)) * float64(src.VoxelSize[i]) maxPtDist[i] = float64(src.MaxPoint.Value(i)) * float64(src.VoxelSize[i]) } // Adjust min and max points for the tileable surface at lowest resolution. var minTiledPt, maxTiledPt dvid.Point3d for i := 0; i < 3; i++ { minInt, _ := math.Modf(minPtDist[i] / loresSize[i]) maxInt, _ := math.Modf(maxPtDist[i] / loresSize[i]) minTileCoord := int32(minInt) maxTileCoord := int32(maxInt) minTiledPt[i] = minTileCoord * DefaultTileSize[i] * loresMag[i] maxTiledPt[i] = (maxTileCoord+1)*DefaultTileSize[i]*loresMag[i] - 1 } sizeVolume := maxTiledPt.Sub(minTiledPt).AddScalar(1) // Setup swappable ExtData buffers (the stitched slices) so we can be generating tiles // at same time we are reading and stitching them. var bufferLock [2]sync.Mutex var sliceBuffers [2]*imageblk.Voxels var bufferNum int // Get the planes we should tile. planes, err := config.GetShapes("planes", ";") if planes == nil { // If no planes are specified, construct imagetile for 3 orthogonal planes. planes = []dvid.DataShape{dvid.XY, dvid.XZ, dvid.YZ} } outF, err := d.putTileFunc(versionID) // sort the tile spec keys to iterate from highest to lowest resolution var sortedKeys []int for scaling, _ := range tileSpec { sortedKeys = append(sortedKeys, int(scaling)) } sort.Ints(sortedKeys) for _, plane := range planes { timedLog := dvid.NewTimeLog() offset := minTiledPt.Duplicate() switch { case plane.Equals(dvid.XY): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling XY image %d x %d pixels\n", width, height) for z := src.MinPoint.Value(2); z <= src.MaxPoint.Value(2); z++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xy]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{2: z}) slice, err := dvid.NewOrthogSlice(dvid.XY, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled XY Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Infof("Read XY Tile @ Z = %d, now tiling...", z) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate XY Tiles") case plane.Equals(dvid.XZ): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling XZ image %d x %d pixels\n", width, height) for y := src.MinPoint.Value(1); y <= src.MaxPoint.Value(1); y++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xz]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{1: y}) slice, err := dvid.NewOrthogSlice(dvid.XZ, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled XZ Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Infof("Read XZ Tile @ Y = %d, now tiling...", y) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate XZ Tiles") case plane.Equals(dvid.YZ): width, height, err := plane.GetSize2D(sizeVolume) if err != nil { return err } dvid.Debugf("Tiling YZ image %d x %d pixels\n", width, height) for x := src.MinPoint.Value(0); x <= src.MaxPoint.Value(0); x++ { server.BlockOnInteractiveRequests("imagetile.ConstructTiles [yz]") sliceLog := dvid.NewTimeLog() offset = offset.Modify(map[uint8]int32{0: x}) slice, err := dvid.NewOrthogSlice(dvid.YZ, offset, dvid.Point2d{width, height}) if err != nil { return err } bufferLock[bufferNum].Lock() sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil) if err != nil { return err } if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil { return err } // Iterate through the different scales, extracting tiles at each resolution. go func(bufferNum int, offset dvid.Point) { defer bufferLock[bufferNum].Unlock() timedLog := dvid.NewTimeLog() for _, key := range sortedKeys { scaling := Scaling(key) levelSpec := tileSpec[scaling] outF, err := d.putTileFunc(versionID) if err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } if int(scaling) < len(tileSpec)-1 { if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil { dvid.Errorf("Error in tiling: %v\n", err) return } } } timedLog.Debugf("Tiled YZ Tile using buffer %d", bufferNum) }(bufferNum, offset) sliceLog.Debugf("Read YZ Tile @ X = %d, now tiling...", x) bufferNum = (bufferNum + 1) % 2 } timedLog.Infof("Total time to generate YZ Tiles") default: dvid.Infof("Skipping request to tile '%s'. Unsupported.", plane) } } return nil }
// Do acts as a switchboard for remote command execution func (c *RPCConnection) Do(cmd datastore.Request, reply *datastore.Response) error { if reply == nil { dvid.Debugf("reply is nil coming in!\n") return nil } if cmd.Name() == "" { return fmt.Errorf("Server error: got empty command!") } switch cmd.Name() { case "help": reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress()) case "shutdown": Shutdown() // Make this process shutdown in a second to allow time for RPC to finish. // TODO -- Better way to do this? log.Printf("DVID server halted due to 'shutdown' command.") reply.Text = fmt.Sprintf("DVID server at %s has been halted.\n", config.RPCAddress()) go func() { time.Sleep(1 * time.Second) os.Exit(0) }() case "types": if len(cmd.Command) == 1 { text := "\nData Types within this DVID Server\n" text += "----------------------------------\n" mapTypes, err := datastore.Types() if err != nil { return fmt.Errorf("Error trying to retrieve data types within this DVID server!") } for url, typeservice := range mapTypes { text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url) } reply.Text = text } else { if len(cmd.Command) != 3 || cmd.Command[2] != "help" { return fmt.Errorf("Unknown types command: %q", cmd.Command) } var typename string cmd.CommandArgs(1, &typename) typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename)) if err != nil { return err } reply.Text = typeservice.Help() } case "repos": var subcommand, alias, description, uuidStr string cmd.CommandArgs(1, &subcommand, &alias, &description, &uuidStr) switch subcommand { case "new": var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } root, err := datastore.NewRepo(alias, description, assign) if err != nil { return err } if err := datastore.SetRepoAlias(root, alias); err != nil { return err } if err := datastore.SetRepoDescription(root, description); err != nil { return err } reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root) default: return fmt.Errorf("Unknown repos command: %q", subcommand) } case "repo": var uuidStr, subcommand string cmd.CommandArgs(1, &uuidStr, &subcommand) uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } switch subcommand { case "new": var typename, dataname string cmd.CommandArgs(3, &typename, &dataname) // Get TypeService typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename)) if err != nil { return err } // Create new data config := cmd.Settings() _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config) if err != nil { return err } reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "branch": cmd.CommandArgs(3, &uuidStr) var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } child, err := datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign) if err != nil { return err } reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "merge": uuids := cmd.CommandArgs(2) parents := make([]dvid.UUID, len(uuids)+1) parents[0] = dvid.UUID(uuid) i := 1 for uuid := range uuids { parents[i] = dvid.UUID(uuid) i++ } child, err := datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree) if err != nil { return err } reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "push": /* var target string cmd.CommandArgs(3, &target) config := cmd.Settings() if err = datastore.Push(repo, target, config); err != nil { return err } reply.Text = fmt.Sprintf("Repo %q pushed to %q\n", repo.RootUUID(), target) */ return fmt.Errorf("push command has been temporarily suspended") default: return fmt.Errorf("Unknown command: %q", cmd) } case "node": var uuidStr, descriptor string cmd.CommandArgs(1, &uuidStr, &descriptor) uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Get the DataService dataname := dvid.InstanceName(descriptor) var subcommand string cmd.CommandArgs(3, &subcommand) dataservice, err := datastore.GetDataByUUID(uuid, dataname) if err != nil { return err } if subcommand == "help" { reply.Text = dataservice.Help() return nil } return dataservice.DoRPC(cmd, reply) default: return fmt.Errorf("Unknown command: '%s'", cmd) } return nil }
// switchboard for remote command execution func handleCommand(cmd *datastore.Request) (reply *datastore.Response, err error) { if cmd.Name() == "" { err = fmt.Errorf("Server error: got empty command!") return } reply = new(datastore.Response) switch cmd.Name() { case "help": reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress()) case "shutdown": dvid.Infof("DVID server halting due to 'shutdown' command.") reply.Text = fmt.Sprintf("DVID server at %s is being shutdown...\n", config.RPCAddress()) // launch goroutine shutdown so we can concurrently return shutdown message to client. go Shutdown() case "types": if len(cmd.Command) == 1 { text := "\nData Types within this DVID Server\n" text += "----------------------------------\n" var mapTypes map[dvid.URLString]datastore.TypeService if mapTypes, err = datastore.Types(); err != nil { err = fmt.Errorf("Error trying to retrieve data types within this DVID server!") return } for url, typeservice := range mapTypes { text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url) } reply.Text = text } else { if len(cmd.Command) != 3 || cmd.Command[2] != "help" { err = fmt.Errorf("Unknown types command: %q", cmd.Command) return } var typename string var typeservice datastore.TypeService cmd.CommandArgs(1, &typename) if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil { return } reply.Text = typeservice.Help() } case "repos": var subcommand string cmd.CommandArgs(1, &subcommand) switch subcommand { case "new": var alias, description string cmd.CommandArgs(2, &alias, &description) config := cmd.Settings() var uuidStr, passcode string var found bool if uuidStr, found, err = config.GetString("uuid"); err != nil { return } var assign *dvid.UUID if !found { assign = nil } else { uuid := dvid.UUID(uuidStr) assign = &uuid } if passcode, found, err = config.GetString("passcode"); err != nil { return } var root dvid.UUID root, err = datastore.NewRepo(alias, description, assign, passcode) if err != nil { return } if err = datastore.SetRepoAlias(root, alias); err != nil { return } if err = datastore.SetRepoDescription(root, description); err != nil { return } reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root) case "delete": var uuidStr, passcode string cmd.CommandArgs(2, &uuidStr, &passcode) var uuid dvid.UUID if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil { return } if err = datastore.DeleteRepo(uuid, passcode); err != nil { return } reply.Text = fmt.Sprintf("Started deletion of repo %s.\n", uuid) default: err = fmt.Errorf("Unknown repos command: %q", subcommand) return } case "repo": var uuidStr, subcommand string cmd.CommandArgs(1, &uuidStr, &subcommand) var uuid dvid.UUID if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil { return } switch subcommand { case "new": var typename, dataname string cmd.CommandArgs(3, &typename, &dataname) // Get TypeService var typeservice datastore.TypeService if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil { return } // Create new data config := cmd.Settings() if _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config); err != nil { return } reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "rename": var name1, name2, passcode string cmd.CommandArgs(3, &name1, &name2, &passcode) oldname := dvid.InstanceName(name1) newname := dvid.InstanceName(name2) // Make sure this instance exists. if _, err = datastore.GetDataByUUIDName(uuid, oldname); err != nil { err = fmt.Errorf("Error trying to rename %q for UUID %s: %v", oldname, uuid, err) return } // Do the rename. if err = datastore.RenameData(uuid, oldname, newname, passcode); err != nil { err = fmt.Errorf("Error renaming data instance %q to %q: %v", oldname, newname, err) return } reply.Text = fmt.Sprintf("Renamed data instance %q to %q from DAG subgraph @ root %s\n", oldname, newname, uuid) case "branch": cmd.CommandArgs(3, &uuidStr) var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } var child dvid.UUID if child, err = datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign); err != nil { return } reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "merge": uuids := cmd.CommandArgs(2) parents := make([]dvid.UUID, len(uuids)+1) parents[0] = dvid.UUID(uuid) i := 1 for uuid := range uuids { parents[i] = dvid.UUID(uuid) i++ } var child dvid.UUID child, err = datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree) if err != nil { return } reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "migrate": var source, oldStoreName string cmd.CommandArgs(3, &source, &oldStoreName) var store dvid.Store store, err = storage.GetStoreByAlias(storage.Alias(oldStoreName)) if err != nil { return } config := cmd.Settings() go func() { if err = datastore.MigrateInstance(uuid, dvid.InstanceName(source), store, config); err != nil { dvid.Errorf("migrate error: %v\n", err) } }() reply.Text = fmt.Sprintf("Started migration of uuid %s data instance %q from old store %q...\n", uuid, source, oldStoreName) case "copy": var source, target string cmd.CommandArgs(3, &source, &target) config := cmd.Settings() go func() { if err = datastore.CopyInstance(uuid, dvid.InstanceName(source), dvid.InstanceName(target), config); err != nil { dvid.Errorf("copy error: %v\n", err) } }() reply.Text = fmt.Sprintf("Started copy of uuid %s data instance %q to %q...\n", uuid, source, target) case "push": var target string cmd.CommandArgs(3, &target) config := cmd.Settings() go func() { if err = datastore.PushRepo(uuid, target, config); err != nil { dvid.Errorf("push error: %v\n", err) } }() reply.Text = fmt.Sprintf("Started push of repo %s to %q...\n", uuid, target) /* case "pull": var target string cmd.CommandArgs(3, &target) config := cmd.Settings() if err = datastore.Pull(uuid, target, config); err != nil { return } reply.Text = fmt.Sprintf("Repo %s pulled from %q\n", uuid, target) */ case "delete": var dataname, passcode string cmd.CommandArgs(3, &dataname, &passcode) // Make sure this instance exists. if _, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(dataname)); err != nil { err = fmt.Errorf("Error trying to delete %q for UUID %s: %v", dataname, uuid, err) return } // Do the deletion. Under hood, modifies metadata immediately and launches async k/v deletion. if err = datastore.DeleteDataByName(uuid, dvid.InstanceName(dataname), passcode); err != nil { err = fmt.Errorf("Error deleting data instance %q: %v", dataname, err) return } reply.Text = fmt.Sprintf("Started deletion of data instance %q from repo with root %s\n", dataname, uuid) default: err = fmt.Errorf("Unknown command: %q", cmd) return } case "node": var uuidStr, descriptor string cmd.CommandArgs(1, &uuidStr, &descriptor) var uuid dvid.UUID if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil { return } // Get the DataService dataname := dvid.InstanceName(descriptor) var subcommand string cmd.CommandArgs(3, &subcommand) var dataservice datastore.DataService if dataservice, err = datastore.GetDataByUUIDName(uuid, dataname); err != nil { return } if subcommand == "help" { reply.Text = dataservice.Help() return } err = dataservice.DoRPC(*cmd, reply) return default: // Check to see if it's a name of a compiled data type, in which case we refer it to the data type. types := datastore.CompiledTypes() for name, typeservice := range types { if name == dvid.TypeString(cmd.Argument(0)) { err = typeservice.Do(*cmd, reply) return } } err = fmt.Errorf("Unknown command: '%s'", *cmd) } return }