// put handles a PUT command-line request. func (d *Data) put(cmd datastore.Request, reply *datastore.Response) error { if len(cmd.Command) < 5 { return fmt.Errorf("The key name must be specified after 'put'") } if len(cmd.Input) == 0 { return fmt.Errorf("No data was passed into standard input") } var uuidStr, dataName, cmdStr, keyStr string cmd.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &keyStr) _, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Store data if !d.Versioned() { // Map everything to root version. versionID, err = datastore.GetRepoRootVersion(versionID) if err != nil { return err } } ctx := datastore.NewVersionedCtx(d, versionID) if err = d.PutData(ctx, keyStr, cmd.Input); err != nil { return fmt.Errorf("Error on put to key %q for keyvalue %q: %v\n", keyStr, d.DataName(), err) } reply.Output = []byte(fmt.Sprintf("Put %d bytes into key %q for keyvalue %q, uuid %s\n", len(cmd.Input), keyStr, d.DataName(), uuidStr)) return nil }
// DoRPC handles the 'generate' command. func (d *Data) DoRPC(request datastore.Request, reply *datastore.Response) error { if request.TypeCommand() != "generate" { return fmt.Errorf("Unknown command. Data instance '%s' [%s] does not support '%s' command.", d.DataName(), d.TypeName(), request.TypeCommand()) } var uuidStr, dataName, cmdStr string request.CommandArgs(1, &uuidStr, &dataName, &cmdStr) // Get the imagetile generation configuration from a file or stdin. var err error var tileSpec TileSpec if request.Input != nil { tileSpec, err = LoadTileSpec(request.Input) if err != nil { return err } } else { config := request.Settings() filename, found, err := config.GetString("filename") if err != nil { return err } if found { configData, err := storage.DataFromFile(filename) if err != nil { return err } tileSpec, err = LoadTileSpec(configData) if err != nil { return err } dvid.Infof("Using tile spec file: %s\n", filename) } else { dvid.Infof("Using default tile generation method since no tile spec file was given...\n") tileSpec, err = d.DefaultTileSpec(uuidStr) if err != nil { return err } } } reply.Text = fmt.Sprintf("Tiling data instance %q @ node %s...\n", dataName, uuidStr) go func() { err := d.ConstructTiles(uuidStr, tileSpec, request) if err != nil { dvid.Errorf("Cannot construct tiles for data instance %q @ node %s: %v\n", dataName, uuidStr, err) } }() return nil }
// ForegroundROI creates a new ROI by determining all non-background blocks. func (d *Data) ForegroundROI(req datastore.Request, reply *datastore.Response) error { if d.Values.BytesPerElement() != 1 { return fmt.Errorf("Foreground ROI command only implemented for 1 byte/voxel data!") } // Parse the request var uuidStr, dataName, cmdStr, destName, backgroundStr string req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &destName, &backgroundStr) // Get the version and repo uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil { return err } // Use existing destination data or a new ROI data. var dest *roi.Data dest, err = roi.GetByUUID(uuid, dvid.InstanceName(destName)) if err != nil { config := dvid.NewConfig() typeservice, err := datastore.TypeServiceByName("roi") if err != nil { return err } dataservice, err := datastore.NewData(uuid, typeservice, dvid.InstanceName(destName), config) if err != nil { return err } var ok bool dest, ok = dataservice.(*roi.Data) if !ok { return fmt.Errorf("Could not create ROI data instance") } } // Asynchronously process the voxels. background, err := dvid.StringToPointNd(backgroundStr, ",") if err != nil { return err } go d.foregroundROI(versionID, dest, background) return nil }
// LoadLocal adds image data to a version node. See HelpMessage for example of // command-line use of "load local". func (d *Data) LoadLocal(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, sourceStr, filename string _ = request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr, &filename) // Get the uuid from a uniquely identifiable string uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return fmt.Errorf("Could not find node with UUID %s: %v", uuidStr, err) } // Load the V3D Raw file. ext := filepath.Ext(filename) switch ext { case ".raw", ".v3draw": default: return fmt.Errorf("Unknown extension '%s' when expected V3D Raw file", ext) } file, err := os.Open(filename) if err != nil { return err } unmarshaler := V3DRawMarshaler{} channels, err := unmarshaler.UnmarshalV3DRaw(file) if err != nil { return err } // Store the metadata d.NumChannels = len(channels) d.Properties.Values = make(dvid.DataValues, d.NumChannels) if d.NumChannels > 0 { reply.Text = fmt.Sprintf("Loaded %s into data '%s': found %d channels\n", d.DataName(), filename, d.NumChannels) reply.Text += fmt.Sprintf(" %s", channels[0]) } else { reply.Text = fmt.Sprintf("Found no channels in file %s\n", filename) return nil } for i, channel := range channels { d.Properties.Values[i] = channel.Voxels.Values()[0] } // Get repo and save it. if err := datastore.SaveDataByUUID(uuid, d); err != nil { return err } // PUT each channel of the file into the datastore using a separate data name. for _, channel := range channels { dvid.Infof("Processing channel %d... \n", channel.channelNum) err = d.PutVoxels(versionID, channel.Voxels, nil) if err != nil { return err } } // Create a RGB composite from the first 3 channels. This is considered to be channel 0 // or can be accessed with the base data name. dvid.Infof("Creating composite image from channels...\n") err = d.storeComposite(versionID, channels) if err != nil { return err } timedLog.Infof("RPC load local '%s' completed", filename) return nil }
// PutLocal adds image data to a version node, altering underlying blocks if the image // intersects the block. // // The image filename glob MUST BE absolute file paths that are visible to the server. // This function is meant for mass ingestion of large data files, and it is inappropriate // to read gigabytes of data just to send it over the network to a local DVID. func (d *Data) PutLocal(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, sourceStr, planeStr, offsetStr string filenames := request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr, &planeStr, &offsetStr) if len(filenames) == 0 { return fmt.Errorf("Need to include at least one file to add: %s", request) } // Get offset offset, err := dvid.StringToPoint(offsetStr, ",") if err != nil { return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err) } // Get list of files to add var addedFiles string if len(filenames) == 1 { addedFiles = filenames[0] } else { addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1) } dvid.Debugf(addedFiles + "\n") // Get plane plane, err := dvid.DataShapeString(planeStr).DataShape() if err != nil { return err } // Get Repo and IDs uuid, versionID, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Load and PUT each image. numSuccessful := 0 for _, filename := range filenames { sliceLog := dvid.NewTimeLog() img, _, err := dvid.GoImageFromFile(filename) if err != nil { return fmt.Errorf("Error after %d images successfully added: %v", numSuccessful, err) } slice, err := dvid.NewOrthogSlice(plane, offset, dvid.RectSize(img.Bounds())) if err != nil { return fmt.Errorf("Unable to determine slice: %v", err) } vox, err := d.NewVoxels(slice, img) if err != nil { return err } storage.FileBytesRead <- len(vox.Data()) if err = d.PutVoxels(versionID, vox, nil); err != nil { return err } sliceLog.Debugf("%s put local %s", d.DataName(), slice) numSuccessful++ offset = offset.Add(dvid.Point3d{0, 0, 1}) } if err := datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } timedLog.Infof("RPC put local (%s) completed", addedFiles) return nil }
// CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error { timedLog := dvid.NewTimeLog() // Parse the request var uuidStr, dataName, cmdStr, grayscaleName, destName string request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName) // Get the version uuid, v, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Log request if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { return err } // Get the grayscale data. dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(grayscaleName)) if err != nil { return err } grayscale, ok := dataservice.(*imageblk.Data) if !ok { return fmt.Errorf("%s is not the name of uint8 data", grayscaleName) } // Create a new rgba8blk data. var compservice datastore.DataService compservice, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(destName)) if err == nil { return fmt.Errorf("Data instance with name %q already exists", destName) } typeService, err := datastore.TypeServiceByName("rgba8blk") if err != nil { return fmt.Errorf("Could not get rgba8 type service from DVID") } config := dvid.NewConfig() compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config) if err != nil { return err } composite, ok := compservice.(*imageblk.Data) if !ok { return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName) } // Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time. wg := new(sync.WaitGroup) op := &compositeOp{grayscale, composite, v} chunkOp := &storage.ChunkOp{op, wg} store, err := d.GetOrderedKeyValueDB() if err != nil { return err } ctx := datastore.NewVersionedCtx(d, v) extents := d.Extents() blockBeg := imageblk.NewTKey(extents.MinIndex) blockEnd := imageblk.NewTKey(extents.MaxIndex) err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk)) wg.Wait() // Set new mapped data to same extents. composite.Properties.Extents = grayscale.Properties.Extents if err := datastore.SaveDataByUUID(uuid, composite); err != nil { dvid.Infof("Could not save new data '%s': %v\n", destName, err) } timedLog.Infof("Created composite of %s and %s", grayscaleName, destName) return nil }
// Do acts as a switchboard for remote command execution func (c *RPCConnection) Do(cmd datastore.Request, reply *datastore.Response) error { if reply == nil { dvid.Debugf("reply is nil coming in!\n") return nil } if cmd.Name() == "" { return fmt.Errorf("Server error: got empty command!") } switch cmd.Name() { case "help": reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress()) case "shutdown": Shutdown() // Make this process shutdown in a second to allow time for RPC to finish. // TODO -- Better way to do this? log.Printf("DVID server halted due to 'shutdown' command.") reply.Text = fmt.Sprintf("DVID server at %s has been halted.\n", config.RPCAddress()) go func() { time.Sleep(1 * time.Second) os.Exit(0) }() case "types": if len(cmd.Command) == 1 { text := "\nData Types within this DVID Server\n" text += "----------------------------------\n" mapTypes, err := datastore.Types() if err != nil { return fmt.Errorf("Error trying to retrieve data types within this DVID server!") } for url, typeservice := range mapTypes { text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url) } reply.Text = text } else { if len(cmd.Command) != 3 || cmd.Command[2] != "help" { return fmt.Errorf("Unknown types command: %q", cmd.Command) } var typename string cmd.CommandArgs(1, &typename) typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename)) if err != nil { return err } reply.Text = typeservice.Help() } case "repos": var subcommand, alias, description, uuidStr string cmd.CommandArgs(1, &subcommand, &alias, &description, &uuidStr) switch subcommand { case "new": var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } root, err := datastore.NewRepo(alias, description, assign) if err != nil { return err } if err := datastore.SetRepoAlias(root, alias); err != nil { return err } if err := datastore.SetRepoDescription(root, description); err != nil { return err } reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root) default: return fmt.Errorf("Unknown repos command: %q", subcommand) } case "repo": var uuidStr, subcommand string cmd.CommandArgs(1, &uuidStr, &subcommand) uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } switch subcommand { case "new": var typename, dataname string cmd.CommandArgs(3, &typename, &dataname) // Get TypeService typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename)) if err != nil { return err } // Create new data config := cmd.Settings() _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config) if err != nil { return err } reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "branch": cmd.CommandArgs(3, &uuidStr) var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } child, err := datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign) if err != nil { return err } reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "merge": uuids := cmd.CommandArgs(2) parents := make([]dvid.UUID, len(uuids)+1) parents[0] = dvid.UUID(uuid) i := 1 for uuid := range uuids { parents[i] = dvid.UUID(uuid) i++ } child, err := datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree) if err != nil { return err } reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "push": /* var target string cmd.CommandArgs(3, &target) config := cmd.Settings() if err = datastore.Push(repo, target, config); err != nil { return err } reply.Text = fmt.Sprintf("Repo %q pushed to %q\n", repo.RootUUID(), target) */ return fmt.Errorf("push command has been temporarily suspended") default: return fmt.Errorf("Unknown command: %q", cmd) } case "node": var uuidStr, descriptor string cmd.CommandArgs(1, &uuidStr, &descriptor) uuid, _, err := datastore.MatchingUUID(uuidStr) if err != nil { return err } // Get the DataService dataname := dvid.InstanceName(descriptor) var subcommand string cmd.CommandArgs(3, &subcommand) dataservice, err := datastore.GetDataByUUID(uuid, dataname) if err != nil { return err } if subcommand == "help" { reply.Text = dataservice.Help() return nil } return dataservice.DoRPC(cmd, reply) default: return fmt.Errorf("Unknown command: '%s'", cmd) } return nil }
// switchboard for remote command execution func handleCommand(cmd *datastore.Request) (reply *datastore.Response, err error) { if cmd.Name() == "" { err = fmt.Errorf("Server error: got empty command!") return } reply = new(datastore.Response) switch cmd.Name() { case "help": reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress()) case "shutdown": dvid.Infof("DVID server halting due to 'shutdown' command.") reply.Text = fmt.Sprintf("DVID server at %s is being shutdown...\n", config.RPCAddress()) // launch goroutine shutdown so we can concurrently return shutdown message to client. go Shutdown() case "types": if len(cmd.Command) == 1 { text := "\nData Types within this DVID Server\n" text += "----------------------------------\n" var mapTypes map[dvid.URLString]datastore.TypeService if mapTypes, err = datastore.Types(); err != nil { err = fmt.Errorf("Error trying to retrieve data types within this DVID server!") return } for url, typeservice := range mapTypes { text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url) } reply.Text = text } else { if len(cmd.Command) != 3 || cmd.Command[2] != "help" { err = fmt.Errorf("Unknown types command: %q", cmd.Command) return } var typename string var typeservice datastore.TypeService cmd.CommandArgs(1, &typename) if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil { return } reply.Text = typeservice.Help() } case "repos": var subcommand string cmd.CommandArgs(1, &subcommand) switch subcommand { case "new": var alias, description string cmd.CommandArgs(2, &alias, &description) config := cmd.Settings() var uuidStr, passcode string var found bool if uuidStr, found, err = config.GetString("uuid"); err != nil { return } var assign *dvid.UUID if !found { assign = nil } else { uuid := dvid.UUID(uuidStr) assign = &uuid } if passcode, found, err = config.GetString("passcode"); err != nil { return } var root dvid.UUID root, err = datastore.NewRepo(alias, description, assign, passcode) if err != nil { return } if err = datastore.SetRepoAlias(root, alias); err != nil { return } if err = datastore.SetRepoDescription(root, description); err != nil { return } reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root) case "delete": var uuidStr, passcode string cmd.CommandArgs(2, &uuidStr, &passcode) var uuid dvid.UUID if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil { return } if err = datastore.DeleteRepo(uuid, passcode); err != nil { return } reply.Text = fmt.Sprintf("Started deletion of repo %s.\n", uuid) default: err = fmt.Errorf("Unknown repos command: %q", subcommand) return } case "repo": var uuidStr, subcommand string cmd.CommandArgs(1, &uuidStr, &subcommand) var uuid dvid.UUID if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil { return } switch subcommand { case "new": var typename, dataname string cmd.CommandArgs(3, &typename, &dataname) // Get TypeService var typeservice datastore.TypeService if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil { return } // Create new data config := cmd.Settings() if _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config); err != nil { return } reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "rename": var name1, name2, passcode string cmd.CommandArgs(3, &name1, &name2, &passcode) oldname := dvid.InstanceName(name1) newname := dvid.InstanceName(name2) // Make sure this instance exists. if _, err = datastore.GetDataByUUIDName(uuid, oldname); err != nil { err = fmt.Errorf("Error trying to rename %q for UUID %s: %v", oldname, uuid, err) return } // Do the rename. if err = datastore.RenameData(uuid, oldname, newname, passcode); err != nil { err = fmt.Errorf("Error renaming data instance %q to %q: %v", oldname, newname, err) return } reply.Text = fmt.Sprintf("Renamed data instance %q to %q from DAG subgraph @ root %s\n", oldname, newname, uuid) case "branch": cmd.CommandArgs(3, &uuidStr) var assign *dvid.UUID if uuidStr == "" { assign = nil } else { u := dvid.UUID(uuidStr) assign = &u } var child dvid.UUID if child, err = datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign); err != nil { return } reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "merge": uuids := cmd.CommandArgs(2) parents := make([]dvid.UUID, len(uuids)+1) parents[0] = dvid.UUID(uuid) i := 1 for uuid := range uuids { parents[i] = dvid.UUID(uuid) i++ } var child dvid.UUID child, err = datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree) if err != nil { return } reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child) datastore.AddToRepoLog(uuid, []string{cmd.String()}) case "migrate": var source, oldStoreName string cmd.CommandArgs(3, &source, &oldStoreName) var store dvid.Store store, err = storage.GetStoreByAlias(storage.Alias(oldStoreName)) if err != nil { return } config := cmd.Settings() go func() { if err = datastore.MigrateInstance(uuid, dvid.InstanceName(source), store, config); err != nil { dvid.Errorf("migrate error: %v\n", err) } }() reply.Text = fmt.Sprintf("Started migration of uuid %s data instance %q from old store %q...\n", uuid, source, oldStoreName) case "copy": var source, target string cmd.CommandArgs(3, &source, &target) config := cmd.Settings() go func() { if err = datastore.CopyInstance(uuid, dvid.InstanceName(source), dvid.InstanceName(target), config); err != nil { dvid.Errorf("copy error: %v\n", err) } }() reply.Text = fmt.Sprintf("Started copy of uuid %s data instance %q to %q...\n", uuid, source, target) case "push": var target string cmd.CommandArgs(3, &target) config := cmd.Settings() go func() { if err = datastore.PushRepo(uuid, target, config); err != nil { dvid.Errorf("push error: %v\n", err) } }() reply.Text = fmt.Sprintf("Started push of repo %s to %q...\n", uuid, target) /* case "pull": var target string cmd.CommandArgs(3, &target) config := cmd.Settings() if err = datastore.Pull(uuid, target, config); err != nil { return } reply.Text = fmt.Sprintf("Repo %s pulled from %q\n", uuid, target) */ case "delete": var dataname, passcode string cmd.CommandArgs(3, &dataname, &passcode) // Make sure this instance exists. if _, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(dataname)); err != nil { err = fmt.Errorf("Error trying to delete %q for UUID %s: %v", dataname, uuid, err) return } // Do the deletion. Under hood, modifies metadata immediately and launches async k/v deletion. if err = datastore.DeleteDataByName(uuid, dvid.InstanceName(dataname), passcode); err != nil { err = fmt.Errorf("Error deleting data instance %q: %v", dataname, err) return } reply.Text = fmt.Sprintf("Started deletion of data instance %q from repo with root %s\n", dataname, uuid) default: err = fmt.Errorf("Unknown command: %q", cmd) return } case "node": var uuidStr, descriptor string cmd.CommandArgs(1, &uuidStr, &descriptor) var uuid dvid.UUID if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil { return } // Get the DataService dataname := dvid.InstanceName(descriptor) var subcommand string cmd.CommandArgs(3, &subcommand) var dataservice datastore.DataService if dataservice, err = datastore.GetDataByUUIDName(uuid, dataname); err != nil { return } if subcommand == "help" { reply.Text = dataservice.Help() return } err = dataservice.DoRPC(*cmd, reply) return default: // Check to see if it's a name of a compiled data type, in which case we refer it to the data type. types := datastore.CompiledTypes() for name, typeservice := range types { if name == dvid.TypeString(cmd.Argument(0)) { err = typeservice.Do(*cmd, reply) return } } err = fmt.Errorf("Unknown command: '%s'", *cmd) } return }