func repoDeleteInstanceHandler(c web.C, w http.ResponseWriter, r *http.Request) {
uuid := c.Env["uuid"].(dvid.UUID)
queryValues := r.URL.Query()
imsure := queryValues.Get("imsure")
if imsure != "true" {
BadRequest(w, r, "Cannot delete instance unless query string 'imsure=true' is present!")
return
}
dataname, ok := c.URLParams["dataname"]
if !ok {
BadRequest(w, r, "Error retrieving data instance name from URL parameters")
return
}
// Make sure this instance exists.
_, err := datastore.GetDataByUUID(uuid, dvid.InstanceName(dataname))
if err != nil {
BadRequest(w, r, "Error trying to delete %q for UUID %s: %v", dataname, uuid, err)
return
}
// Do the deletion. Under hood, modifies metadata immediately and launches async k/v deletion.
if err := datastore.DeleteDataByUUID(uuid, dvid.InstanceName(dataname)); err != nil {
BadRequest(w, r, "Error deleting data instance %q: %v", dataname, err)
return
}
// Just respond that deletion was successfully started
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"result": "Started deletion of data instance %q from repo with root %s"}`,
dataname, uuid)
}
// instanceSelector retrieves the data instance given its complete string name and
// forwards the request to that instance's HTTP handler.
func instanceSelector(c *web.C, h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
var err error
dataname := dvid.InstanceName(c.URLParams["dataname"])
uuid, ok := c.Env["uuid"].(dvid.UUID)
if !ok {
msg := fmt.Sprintf("Bad format for UUID %q\n", c.Env["uuid"])
BadRequest(w, r, msg)
return
}
data, err := datastore.GetDataByUUID(uuid, dataname)
if err != nil {
BadRequest(w, r, err)
return
}
v, err := datastore.VersionFromUUID(uuid)
if err != nil {
BadRequest(w, r, err)
}
ctx := datastore.NewVersionedCtx(data, v)
// Handle DVID-wide query string commands like non-interactive call designations
queryValues := r.URL.Query()
// All HTTP requests are interactive so let server tally request.
interactive := queryValues.Get("interactive")
if interactive == "" || (interactive != "false" && interactive != "0") {
GotInteractiveRequest()
}
// TODO: setup routing for data instances as well.
data.ServeHTTP(uuid, ctx, w, r)
}
return http.HandlerFunc(fn)
}
func repoDeleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
uuid := c.Env["uuid"].(dvid.UUID)
queryValues := r.URL.Query()
imsure := queryValues.Get("imsure")
if imsure != "true" {
BadRequest(w, r, "Cannot delete instance unless query string 'imsure=true' is present!")
return
}
dataname, ok := c.URLParams["dataname"]
if !ok {
BadRequest(w, r, "Error in retrieving data instance name from URL parameters")
return
}
// Do the deletion asynchronously since they can take a very long time.
go func() {
if err := datastore.DeleteDataByUUID(uuid, dvid.InstanceName(dataname)); err != nil {
dvid.Errorf("Error in deleting data instance %q: %v", dataname, err)
}
}()
// Just respond that deletion was successfully started
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"result": "Started deletion of data instance %q from repo with root %s"}`,
dataname, uuid)
}
// DataBySpec returns a ROI Data based on a string specification of the form
// "<roiname>,<uuid>". If the given string is not parsable, the "found" return value is false.
func DataBySpec(spec string) (d *Data, v dvid.VersionID, found bool, err error) {
roispec := strings.Split(spec, ",")
if len(roispec) != 2 {
err = fmt.Errorf("Expect ROI filters to have format %q, but got %q", "roi:<roiname>,<uuid>", spec)
return
}
roiName := dvid.InstanceName(roispec[0])
_, v, err = datastore.MatchingUUID(roispec[1])
if err != nil {
return
}
var data datastore.DataService
data, err = datastore.GetDataByVersionName(v, roiName)
if err != nil {
return
}
var ok bool
d, ok = data.(*Data)
if !ok {
err = fmt.Errorf("Data instance %q is not ROI instance", roiName)
return
}
found = true
return
}
func repoNewDataHandler(c web.C, w http.ResponseWriter, r *http.Request) {
uuid := c.Env["uuid"].(dvid.UUID)
config := dvid.NewConfig()
if err := config.SetByJSON(r.Body); err != nil {
BadRequest(w, r, fmt.Sprintf("Error decoding POSTed JSON config for 'new': %v", err))
return
}
// Make sure that the passed configuration has data type and instance name.
typename, found, err := config.GetString("typename")
if !found || err != nil {
BadRequest(w, r, "POST on repo endpoint requires specification of valid 'typename'")
return
}
dataname, found, err := config.GetString("dataname")
if !found || err != nil {
BadRequest(w, r, "POST on repo endpoint requires specification of valid 'dataname'")
return
}
typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename))
if err != nil {
BadRequest(w, r, err)
return
}
_, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config)
if err != nil {
BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "{%q: 'Added %s [%s] to node %s'}", "result", dataname, typename, uuid)
}
// ForegroundROI creates a new ROI by determining all non-background blocks.
func (d *Data) ForegroundROI(req datastore.Request, reply *datastore.Response) error {
if d.Values.BytesPerElement() != 1 {
return fmt.Errorf("Foreground ROI command only implemented for 1 byte/voxel data!")
}
// Parse the request
var uuidStr, dataName, cmdStr, destName, backgroundStr string
req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &destName, &backgroundStr)
// Get the version and repo
uuid, versionID, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil {
return err
}
// Use existing destination data or a new ROI data.
var dest *roi.Data
dest, err = roi.GetByUUID(uuid, dvid.InstanceName(destName))
if err != nil {
config := dvid.NewConfig()
typeservice, err := datastore.TypeServiceByName("roi")
if err != nil {
return err
}
dataservice, err := datastore.NewData(uuid, typeservice, dvid.InstanceName(destName), config)
if err != nil {
return err
}
var ok bool
dest, ok = dataservice.(*roi.Data)
if !ok {
return fmt.Errorf("Could not create ROI data instance")
}
}
// Asynchronously process the voxels.
background, err := dvid.StringToPointNd(backgroundStr, ",")
if err != nil {
return err
}
go d.foregroundROI(versionID, dest, background)
return nil
}
func GetTestDataContext(uuid dvid.UUID, name string, instanceID dvid.InstanceID) *DataContext {
versionID, found := testUUIDToVersion[uuid]
if !found {
return nil
}
data := &testData{uuid, dvid.InstanceName(name), instanceID}
return NewDataContext(data, versionID)
}
// ServeTile returns a tile with appropriate Content-Type set.
func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter,
r *http.Request, parts []string) error {
if d.Levels == nil || len(d.Levels) == 0 {
return ErrNoMetadataSet
}
tileReq, err := d.parseTileReq(r, parts)
queryValues := r.URL.Query()
noblanksStr := dvid.InstanceName(queryValues.Get("noblanks"))
var noblanks bool
if noblanksStr == "true" {
noblanks = true
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
data, err := d.getTileData(ctx, tileReq)
if err != nil {
server.BadRequest(w, r, err)
return err
}
if data == nil {
if noblanks {
http.NotFound(w, r)
return nil
}
img, err := d.getBlankTileImage(uuid, tileReq)
if err != nil {
return err
}
return dvid.WriteImageHttp(w, img, formatStr)
}
switch d.Encoding {
case LZ4:
var img dvid.Image
if err := img.Deserialize(data); err != nil {
return err
}
data, err = img.GetPNG()
w.Header().Set("Content-type", "image/png")
case PNG:
w.Header().Set("Content-type", "image/png")
case JPG:
w.Header().Set("Content-type", "image/jpeg")
}
if err != nil {
server.BadRequest(w, r, err)
return err
}
if _, err = w.Write(data); err != nil {
return err
}
return nil
}
func makeGrayscale(uuid dvid.UUID, t *testing.T, name string) *Data {
config := dvid.NewConfig()
dataservice, err := datastore.NewData(uuid, grayscaleT, dvid.InstanceName(name), config)
if err != nil {
t.Errorf("Unable to create grayscale instance %q: %v\n", name, err)
}
grayscale, ok := dataservice.(*Data)
if !ok {
t.Errorf("Can't cast uint8blk data service into Data\n")
}
return grayscale
}
func setupGroupcache(config GroupcacheConfig) error {
if config.GB == 0 {
return nil
}
var cacheBytes int64
cacheBytes = int64(config.GB) << 30
pool := groupcache.NewHTTPPool(config.Host)
if pool != nil {
dvid.Infof("Initializing groupcache with %d GB at %s...\n", config.GB, config.Host)
manager.gcache.cache = groupcache.NewGroup("immutable", cacheBytes, groupcache.GetterFunc(
func(c groupcache.Context, key string, dest groupcache.Sink) error {
// Use KeyValueDB defined as context.
gctx, ok := c.(GroupcacheCtx)
if !ok {
return fmt.Errorf("bad groupcache context: expected GroupcacheCtx, got %v", c)
}
// First four bytes of key is instance ID to isolate groupcache collisions.
tk := TKey(key[4:])
data, err := gctx.KeyValueDB.Get(gctx.Context, tk)
if err != nil {
return err
}
return dest.SetBytes(data)
}))
manager.gcache.supported = make(map[dvid.DataSpecifier]struct{})
for _, dataspec := range config.Instances {
name := strings.Trim(dataspec, "\"")
parts := strings.Split(name, ":")
switch len(parts) {
case 2:
dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
manager.gcache.supported[dataid] = struct{}{}
default:
dvid.Errorf("bad data instance specification %q given for groupcache support in config file\n", dataspec)
}
}
// If we have additional peers, add them and start a listener via the HTTP port.
if len(config.Peers) > 0 {
peers := []string{config.Host}
peers = append(peers, config.Peers...)
pool.Set(peers...)
dvid.Infof("Groupcache configuration has %d peers in addition to local host.\n", len(config.Peers))
dvid.Infof("Starting groupcache HTTP server on %s\n", config.Host)
http.ListenAndServe(config.Host, http.HandlerFunc(pool.ServeHTTP))
}
}
return nil
}
// ParseFilterSpec returns the specified ROI instance name and version within a FilterSpec.
// Currently, only one ROI can be specified in a FilterSpec. Multiple ROIs should use a
// different FilterSpec like "intersect" instead of "roi".
func ParseFilterSpec(spec storage.FilterSpec) (name dvid.InstanceName, v dvid.VersionID, found bool, err error) {
var filterval string
filterval, found = spec.GetFilterSpec("roi")
if !found {
return
}
roispec := strings.Split(filterval, ",")
if len(roispec) != 2 {
err = fmt.Errorf("bad ROI spec: %s", filterval)
return
}
name = dvid.InstanceName(roispec[0])
_, v, err = datastore.MatchingUUID(roispec[1])
return
}
// Make a copy of a repository, customizing it via config.
// TODO -- modify data instance properties based on filters.
func (r *repoT) customize(v dvid.VersionID, config dvid.Config) (*repoT, rpc.Transmit, error) {
// Since we can have names separated by commas, split them
namesStr, found, err := config.GetString("data")
if err != nil {
return nil, rpc.TransmitUnknown, err
}
var datanames dvid.InstanceNames
if found {
for _, name := range strings.Split(namesStr, ",") {
datanames = append(datanames, dvid.InstanceName(strings.TrimSpace(name)))
}
}
// Check the transmission behavior: all, flatten, or deltas.
var versions map[dvid.VersionID]struct{}
transmitStr, found, err := config.GetString("transmit")
if err != nil {
return nil, rpc.TransmitUnknown, err
}
if !found {
transmitStr = "all"
}
var transmit rpc.Transmit
switch transmitStr {
case "flatten":
transmit = rpc.TransmitFlatten
versions = map[dvid.VersionID]struct{}{
v: struct{}{},
}
case "all":
transmit = rpc.TransmitAll
versions = r.versionSet()
case "branch":
transmit = rpc.TransmitBranch
versions = r.versionSet()
default:
return nil, rpc.TransmitUnknown, fmt.Errorf("unknown transmit %s", transmitStr)
}
// Make a copy filtering by allowed data instances.
r.RLock()
defer r.RUnlock()
dup, err := r.duplicate(versions, datanames)
return dup, transmit, err
}
// SetSyncByJSON takes a JSON object of sync names and UUID, and creates the sync graph
// and sets the data instance's sync. If replace is false (default), the new sync
// is appended to the current syncs.
func SetSyncByJSON(d dvid.Data, uuid dvid.UUID, replace bool, in io.ReadCloser) error {
if manager == nil {
return ErrManagerNotInitialized
}
jsonData := make(map[string]string)
decoder := json.NewDecoder(in)
if err := decoder.Decode(&jsonData); err != nil && err != io.EOF {
return fmt.Errorf("Malformed JSON request in sync request: %v", err)
}
syncedCSV, ok := jsonData["sync"]
if !ok {
return fmt.Errorf("Could not find 'sync' value in POSTed JSON to sync request.")
}
syncedNames := strings.Split(syncedCSV, ",")
if len(syncedNames) == 0 || (len(syncedNames) == 1 && syncedNames[0] == "") {
syncedNames = []string{}
}
if len(syncedNames) == 0 && !replace {
dvid.Infof("Ignored attempt to append no syncs to instance %q.\n", d.DataName())
return nil
}
// Make sure all synced names currently exist under this UUID, then transform to data UUIDs.
syncs := make(dvid.UUIDSet)
for _, name := range syncedNames {
data, err := GetDataByUUIDName(uuid, dvid.InstanceName(name))
if err != nil {
return err
}
syncs[data.DataUUID()] = struct{}{}
}
if err := SetSyncData(d, syncs, replace); err != nil {
return err
}
return nil
}
// NewData returns a pointer to labelvol data.
func NewData(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
// Make sure we have a valid labelvol source
s, found, err := c.GetString("Source")
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("Cannot make labelsz data without valid 'Source' specifying an associated labelvol.")
}
srcname := dvid.InstanceName(s)
if _, err = labelvol.GetByUUID(uuid, srcname); err != nil {
return nil, err
}
c.Set("sync", s) // This will set base data sync list
// Initialize the Data for this data type
basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
if err != nil {
return nil, err
}
return &Data{basedata, srcname}, nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
// --- Don't time labelgraph ops because they are very small and frequent.
// --- TODO -- Implement monitoring system that aggregates logged ops instead of
// ----------- printing out each one.
// timedLog := dvid.NewTimeLog()
db, err := storage.GraphStore()
if err != nil {
server.BadRequest(w, r, err)
return
}
// make sure transaction log is created
d.initializeLog()
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts) < 4 {
server.BadRequest(w, r, "No resource specified in URI")
return
}
method := strings.ToLower(r.Method)
if method == "put" {
server.BadRequest(w, r, "PUT requests not supported")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "subgraph":
// disable json schema validation (will speedup POST command)
queryValues := r.URL.Query()
disableSchemaT := dvid.InstanceName(queryValues.Get("unsafe"))
disableSchema := false
if len(disableSchemaT) != 0 && disableSchemaT == "true" {
disableSchema = true
}
labelgraph, err := d.ExtractGraph(r, disableSchema)
if err != nil {
server.BadRequest(w, r, err)
return
}
err = d.handleSubgraphBulk(ctx, db, w, labelgraph, method)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "neighbors":
if method != "get" {
server.BadRequest(w, r, "Only supports GETs")
return
}
err := d.handleNeighbors(ctx, db, w, parts[4:])
if err != nil {
server.BadRequest(w, r, err)
return
}
case "merge":
if method != "post" {
server.BadRequest(w, r, "Only supports POSTs")
return
}
labelgraph, err := d.ExtractGraph(r, false)
if err != nil {
server.BadRequest(w, r, err)
return
}
err = d.handleMerge(ctx, db, w, labelgraph)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "weight":
if method != "post" {
server.BadRequest(w, r, "Only supports POSTs")
return
}
labelgraph, err := d.ExtractGraph(r, false)
if err != nil {
server.BadRequest(w, r, err)
return
}
err = d.handleWeightUpdate(ctx, db, w, labelgraph)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "propertytransaction":
err := d.handlePropertyTransaction(ctx, db, w, r, parts[4:], method)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "property":
err := d.handleProperty(ctx, db, w, r, parts[4:], method)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "undomerge":
// not supported until transaction history is supported
server.BadRequest(w, r, "undomerge not yet implemented")
return
default:
server.BadRequest(w, r, "%s not found", parts[3])
return
}
//timedLog.Infof("Successful labelgraph op %s on %q", parts[3], d.DataName())
}
// Do acts as a switchboard for remote command execution
func (c *RPCConnection) Do(cmd datastore.Request, reply *datastore.Response) error {
if reply == nil {
dvid.Debugf("reply is nil coming in!\n")
return nil
}
if cmd.Name() == "" {
return fmt.Errorf("Server error: got empty command!")
}
switch cmd.Name() {
case "help":
reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress())
case "shutdown":
Shutdown()
// Make this process shutdown in a second to allow time for RPC to finish.
// TODO -- Better way to do this?
log.Printf("DVID server halted due to 'shutdown' command.")
reply.Text = fmt.Sprintf("DVID server at %s has been halted.\n", config.RPCAddress())
go func() {
time.Sleep(1 * time.Second)
os.Exit(0)
}()
case "types":
if len(cmd.Command) == 1 {
text := "\nData Types within this DVID Server\n"
text += "----------------------------------\n"
mapTypes, err := datastore.Types()
if err != nil {
return fmt.Errorf("Error trying to retrieve data types within this DVID server!")
}
for url, typeservice := range mapTypes {
text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url)
}
reply.Text = text
} else {
if len(cmd.Command) != 3 || cmd.Command[2] != "help" {
return fmt.Errorf("Unknown types command: %q", cmd.Command)
}
var typename string
cmd.CommandArgs(1, &typename)
typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename))
if err != nil {
return err
}
reply.Text = typeservice.Help()
}
case "repos":
var subcommand, alias, description, uuidStr string
cmd.CommandArgs(1, &subcommand, &alias, &description, &uuidStr)
switch subcommand {
case "new":
var assign *dvid.UUID
if uuidStr == "" {
assign = nil
} else {
u := dvid.UUID(uuidStr)
assign = &u
}
root, err := datastore.NewRepo(alias, description, assign)
if err != nil {
return err
}
if err := datastore.SetRepoAlias(root, alias); err != nil {
return err
}
if err := datastore.SetRepoDescription(root, description); err != nil {
return err
}
reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root)
default:
return fmt.Errorf("Unknown repos command: %q", subcommand)
}
case "repo":
var uuidStr, subcommand string
cmd.CommandArgs(1, &uuidStr, &subcommand)
uuid, _, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
switch subcommand {
case "new":
var typename, dataname string
cmd.CommandArgs(3, &typename, &dataname)
// Get TypeService
typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename))
if err != nil {
return err
}
// Create new data
config := cmd.Settings()
_, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config)
if err != nil {
return err
}
reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid)
datastore.AddToRepoLog(uuid, []string{cmd.String()})
case "branch":
cmd.CommandArgs(3, &uuidStr)
var assign *dvid.UUID
if uuidStr == "" {
assign = nil
} else {
u := dvid.UUID(uuidStr)
assign = &u
}
child, err := datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign)
if err != nil {
return err
}
reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid)
datastore.AddToRepoLog(uuid, []string{cmd.String()})
case "merge":
uuids := cmd.CommandArgs(2)
parents := make([]dvid.UUID, len(uuids)+1)
parents[0] = dvid.UUID(uuid)
i := 1
for uuid := range uuids {
parents[i] = dvid.UUID(uuid)
i++
}
child, err := datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree)
if err != nil {
return err
}
reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child)
datastore.AddToRepoLog(uuid, []string{cmd.String()})
case "push":
/*
var target string
cmd.CommandArgs(3, &target)
config := cmd.Settings()
if err = datastore.Push(repo, target, config); err != nil {
return err
}
reply.Text = fmt.Sprintf("Repo %q pushed to %q\n", repo.RootUUID(), target)
*/
return fmt.Errorf("push command has been temporarily suspended")
default:
return fmt.Errorf("Unknown command: %q", cmd)
}
case "node":
var uuidStr, descriptor string
cmd.CommandArgs(1, &uuidStr, &descriptor)
uuid, _, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
// Get the DataService
dataname := dvid.InstanceName(descriptor)
var subcommand string
cmd.CommandArgs(3, &subcommand)
dataservice, err := datastore.GetDataByUUID(uuid, dataname)
if err != nil {
return err
}
if subcommand == "help" {
reply.Text = dataservice.Help()
return nil
}
return dataservice.DoRPC(cmd, reply)
default:
return fmt.Errorf("Unknown command: '%s'", cmd)
}
return nil
}
// instanceSelector retrieves the data instance given its complete string name and
// forwards the request to that instance's HTTP handler.
func instanceSelector(c *web.C, h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
if httpUnavailable(w) {
return
}
var err error
dataname := dvid.InstanceName(c.URLParams["dataname"])
uuid, ok := c.Env["uuid"].(dvid.UUID)
if !ok {
msg := fmt.Sprintf("Bad format for UUID %q\n", c.Env["uuid"])
BadRequest(w, r, msg)
return
}
data, err := datastore.GetDataByUUIDName(uuid, dataname)
if err != nil {
BadRequest(w, r, err)
return
}
v, err := datastore.VersionFromUUID(uuid)
if err != nil {
BadRequest(w, r, err)
return
}
if data.Versioned() {
// Make sure we aren't trying mutable methods on committed nodes.
locked, err := datastore.LockedUUID(uuid)
if err != nil {
BadRequest(w, r, err)
return
}
if locked && data.IsMutationRequest(r.Method, c.URLParams["keyword"]) {
BadRequest(w, r, "Cannot do %s on endpoint %q of locked node %s", r.Method, c.URLParams["keyword"], uuid)
return
}
} else {
// Map everything to root version.
v, err = datastore.GetRepoRootVersion(v)
if err != nil {
BadRequest(w, r, err)
return
}
}
ctx := datastore.NewVersionedCtx(data, v)
// Also set the web request information in case logging needs it downstream.
ctx.SetRequestID(middleware.GetReqID(*c))
// Handle DVID-wide query string commands like non-interactive call designations
queryStrings := r.URL.Query()
// All HTTP requests are interactive so let server tally request.
interactive := queryStrings.Get("interactive")
if interactive == "" || (interactive != "false" && interactive != "0") {
GotInteractiveRequest()
}
// TODO: setup routing for data instances as well.
if config != nil && config.AllowTiming() {
w.Header().Set("Timing-Allow-Origin", "*")
}
data.ServeHTTP(uuid, ctx, w, r)
}
return http.HandlerFunc(fn)
}
// handleTileReq returns a tile with appropriate Content-Type set.
func (d *Data) handleTileReq(w http.ResponseWriter, r *http.Request, parts []string) error {
if len(parts) < 7 {
return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
}
planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
queryStrings := r.URL.Query()
var noblanks bool
noblanksStr := dvid.InstanceName(queryStrings.Get("noblanks"))
if noblanksStr == "true" {
noblanks = true
}
var tilesize int32 = DefaultTileSize
tileSizeStr := queryStrings.Get("tilesize")
if tileSizeStr != "" {
tilesizeInt, err := strconv.Atoi(tileSizeStr)
if err != nil {
return err
}
tilesize = int32(tilesizeInt)
}
size := dvid.Point2d{tilesize, tilesize}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
if formatStr == "" {
formatStr = DefaultTileFormat
}
// Parse the tile specification
plane := dvid.DataShapeString(planeStr)
shape, err := plane.DataShape()
if err != nil {
err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
server.BadRequest(w, r, err)
return err
}
scale, err := strconv.ParseUint(scalingStr, 10, 8)
if err != nil {
err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
server.BadRequest(w, r, err)
return err
}
tileCoord, err := dvid.StringToPoint(coordStr, "_")
if err != nil {
err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
server.BadRequest(w, r, err)
return err
}
// Convert tile coordinate to offset.
var ox, oy, oz int32
switch {
case shape.Equals(dvid.XY):
ox = tileCoord.Value(0) * tilesize
oy = tileCoord.Value(1) * tilesize
oz = tileCoord.Value(2)
case shape.Equals(dvid.XZ):
ox = tileCoord.Value(0) * tilesize
oy = tileCoord.Value(1)
oz = tileCoord.Value(2) * tilesize
case shape.Equals(dvid.YZ):
ox = tileCoord.Value(0)
oy = tileCoord.Value(1) * tilesize
oz = tileCoord.Value(2) * tilesize
default:
return fmt.Errorf("Unknown tile orientation: %s", shape)
}
// Determine how this request sits in the available scaled volumes.
geom, err := d.GetGoogleSubvolGeom(Scaling(scale), shape, dvid.Point3d{ox, oy, oz}, size)
if err != nil {
server.BadRequest(w, r, err)
return err
}
// Send the tile.
return d.serveTile(w, r, geom, formatStr, noblanks)
}
// ServeTile returns a tile with appropriate Content-Type set.
func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter,
r *http.Request, parts []string) error {
if len(parts) < 7 {
return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
}
planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
queryValues := r.URL.Query()
noblanksStr := dvid.InstanceName(queryValues.Get("noblanks"))
var noblanks bool
if noblanksStr == "true" {
noblanks = true
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
// Construct the index for this tile
plane := dvid.DataShapeString(planeStr)
shape, err := plane.DataShape()
if err != nil {
err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
server.BadRequest(w, r, err)
return err
}
scaling, err := strconv.ParseUint(scalingStr, 10, 8)
if err != nil {
err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
server.BadRequest(w, r, err)
return err
}
tileCoord, err := dvid.StringToPoint(coordStr, "_")
if err != nil {
err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
server.BadRequest(w, r, err)
return err
}
if tileCoord.NumDims() != 3 {
err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr)
server.BadRequest(w, r, err)
return err
}
indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)}
data, err := d.getTileData(ctx, shape, Scaling(scaling), indexZYX)
if err != nil {
server.BadRequest(w, r, err)
return err
}
if data == nil {
if noblanks {
http.NotFound(w, r)
return nil
}
img, err := d.getBlankTileImage(uuid, shape, Scaling(scaling))
if err != nil {
return err
}
return dvid.WriteImageHttp(w, img, formatStr)
}
switch d.Encoding {
case LZ4:
var img dvid.Image
if err := img.Deserialize(data); err != nil {
return err
}
data, err = img.GetPNG()
w.Header().Set("Content-type", "image/png")
case PNG:
w.Header().Set("Content-type", "image/png")
case JPG:
w.Header().Set("Content-type", "image/jpeg")
}
if err != nil {
server.BadRequest(w, r, err)
return err
}
if _, err = w.Write(data); err != nil {
return err
}
return nil
}
// NewData returns a pointer to new tile data with default values.
func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) {
// Make sure we have a valid DataService source
sourcename, found, err := c.GetString("Source")
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("Cannot make imagetile data without valid 'Source' setting.")
}
// See if we want placeholder imagetile.
placeholder, found, err := c.GetBool("Placeholder")
if err != nil {
return nil, err
}
// Determine encoding for tile storage and this dictates what kind of compression we use.
encoding, found, err := c.GetString("Format")
if err != nil {
return nil, err
}
format := PNG
if found {
switch strings.ToLower(encoding) {
case "lz4":
format = LZ4
case "png":
format = PNG
case "jpg":
format = JPG
default:
return nil, fmt.Errorf("Unknown encoding specified: '%s' (should be 'lz4', 'png', or 'jpg'", encoding)
}
}
// Compression is determined by encoding. Inform user if there's a discrepancy.
var compression string
switch format {
case LZ4:
compression = "lz4"
case PNG:
compression = "none"
case JPG:
compression = "none"
}
compressConfig, found, err := c.GetString("Compression")
if err != nil {
return nil, err
}
if found && strings.ToLower(compressConfig) != compression {
return nil, fmt.Errorf("Conflict between specified compression '%s' and format '%s'. Suggest not dictating compression.",
compressConfig, encoding)
}
c.Set("Compression", compression)
// Initialize the imagetile data
basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
if err != nil {
return nil, err
}
data := &Data{
Data: basedata,
Properties: Properties{
Source: dvid.InstanceName(sourcename),
Placeholder: placeholder,
Encoding: format,
},
}
return data, nil
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
// TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints.
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
case "delete":
default:
server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *imageblk.ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(imageblk.ROI)
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "post" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "label":
// GET <api URL>/node/<UUID>/<data name>/label/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)
case "labels":
// POST <api URL>/node/<UUID>/<data name>/labels
if action != "post" {
server.BadRequest(w, r, "Batch labels query must be a POST request")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.")
return
}
var coords []dvid.Point3d
if err := json.Unmarshal(data, &coords); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "[")
sep := false
for _, coord := range coords {
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
if sep {
fmt.Fprintf(w, ",")
}
fmt.Fprintf(w, "%d", label)
sep = true
}
fmt.Fprintf(w, "]")
timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))
case "blocks":
// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "delete" {
server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
return
}
if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)
case "pseudocolor":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
lbl, err := d.NewLabels(slice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
// Convert to pseudocolor
pseudoColor, err := colorImage(img)
if err != nil {
server.BadRequest(w, r, err)
return
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
return
}
case "raw", "isotropic":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
lbl, err := d.NewLabels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
compression := queryValues.Get("compression")
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
lbl, err := d.NewLabels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
switch compression {
case "":
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "lz4":
compressed := make([]byte, lz4.CompressBound(data))
var n, outSize int
if outSize, err = lz4.Compress(data, compressed); err != nil {
server.BadRequest(w, r, err)
return
}
compressed = compressed[:outSize]
if n, err = w.Write(compressed); err != nil {
server.BadRequest(w, r, err)
return
}
if n != outSize {
errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
dvid.Errorf(errmsg)
server.BadRequest(w, r, errmsg)
return
}
case "gzip":
gw := gzip.NewWriter(w)
if _, err = gw.Write(data); err != nil {
server.BadRequest(w, r, err)
return
}
if err = gw.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
} else {
if isotropic {
server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
var data []byte
switch compression {
case "":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read 3d uncompressed POST")
case "lz4":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
if len(data) == 0 {
server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
return
}
tlog.Debugf("read 3d lz4 POST")
tlog = dvid.NewTimeLog()
uncompressed := make([]byte, subvol.NumVoxels()*8)
err = lz4.Uncompress(data, uncompressed)
if err != nil {
server.BadRequest(w, r, err)
return
}
data = uncompressed
tlog.Debugf("uncompressed 3d lz4 POST")
case "gzip":
tlog := dvid.NewTimeLog()
gr, err := gzip.NewReader(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err = ioutil.ReadAll(gr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err = gr.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read and uncompress 3d gzip POST")
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
lbl, err := d.NewLabels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
return
}
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.",
parts[3], d.DataName())
}
}
func (d *Data) ModifyConfig(config dvid.Config) error {
// Set compression for this instance
s, found, err := config.GetString("Compression")
if err != nil {
return err
}
if found {
format := strings.ToLower(s)
switch format {
case "none":
d.compression, _ = dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
case "snappy":
d.compression, _ = dvid.NewCompression(dvid.Snappy, dvid.DefaultCompression)
case "lz4":
d.compression, _ = dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
case "gzip":
d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.DefaultCompression)
default:
// Check for gzip + compression level
parts := strings.Split(format, ":")
if len(parts) == 2 && parts[0] == "gzip" {
level, err := strconv.Atoi(parts[1])
if err != nil {
return fmt.Errorf("Unable to parse gzip compression level ('%d'). Should be 'gzip:<level>'.", parts[1])
}
d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.CompressionLevel(level))
} else {
return fmt.Errorf("Illegal compression specified: %s", s)
}
}
}
// Set checksum for this instance
s, found, err = config.GetString("Checksum")
if err != nil {
return err
}
if found {
checksum := strings.ToLower(s)
switch checksum {
case "none":
d.checksum = dvid.NoChecksum
case "crc32":
d.checksum = dvid.CRC32
default:
return fmt.Errorf("Illegal checksum specified: %s", s)
}
}
// Set data instances for syncing.
s, found, err = config.GetString("sync")
if err != nil {
return err
}
if found {
names := strings.Split(s, ",")
if len(names) > 0 {
for _, name := range names {
d.syncs = append(d.syncs, dvid.InstanceName(name))
}
}
}
// Set versioning
s, found, err = config.GetString("Versioned")
if err != nil {
return err
}
if found {
versioned := strings.ToLower(s)
switch versioned {
case "false", "0":
d.unversioned = true
case "true", "1":
d.unversioned = false
default:
return fmt.Errorf("Illegal setting for 'versioned' (needs to be 'false', '0', 'true', or '1'): %s", s)
}
}
return nil
}
// Initialize the storage systems. Returns a bool + error where the bool is
// true if the metadata store is newly created and needs initialization.
// The map of store configurations should be keyed by either a datatype name,
// "default", or "metadata".
func Initialize(cmdline dvid.Config, backend *Backend) (createdMetadata bool, err error) {
dvid.Infof("backend:\n%v\n", *backend)
// Open all the backend stores
manager.stores = make(map[Alias]dvid.Store, len(backend.Stores))
var gotDefault, gotMetadata, createdDefault, lastCreated bool
var lastStore dvid.Store
for alias, dbconfig := range backend.Stores {
var store dvid.Store
for dbalias, db := range manager.stores {
if db.Equal(dbconfig) {
return false, fmt.Errorf("Store %q configuration is duplicate of store %q", alias, dbalias)
}
}
store, created, err := NewStore(dbconfig)
if err != nil {
return false, fmt.Errorf("bad store %q: %v", alias, err)
}
if alias == backend.Metadata {
gotMetadata = true
createdMetadata = created
manager.metadataStore = store
}
if alias == backend.Default {
gotDefault = true
createdDefault = created
manager.defaultStore = store
}
manager.stores[alias] = store
lastStore = store
lastCreated = created
}
// Return if we don't have default or metadata stores. Should really be caught
// at configuration loading, but here as well as double check.
if !gotDefault {
if len(backend.Stores) == 1 {
manager.defaultStore = lastStore
createdDefault = lastCreated
} else {
return false, fmt.Errorf("either backend.default or a single store must be set in configuration TOML file")
}
}
if !gotMetadata {
manager.metadataStore = manager.defaultStore
createdMetadata = createdDefault
}
dvid.Infof("Default store: %s\n", manager.defaultStore)
dvid.Infof("Metadata store: %s\n", manager.metadataStore)
// Setup the groupcache if specified.
err = setupGroupcache(backend.Groupcache)
if err != nil {
return
}
// Make all data instance or datatype-specific store assignments.
manager.instanceStore = make(map[dvid.DataSpecifier]dvid.Store)
manager.datatypeStore = make(map[dvid.TypeString]dvid.Store)
for dataspec, alias := range backend.Mapping {
if dataspec == "default" || dataspec == "metadata" {
continue
}
store, found := manager.stores[alias]
if !found {
err = fmt.Errorf("bad backend store alias: %q -> %q", dataspec, alias)
return
}
// Cache the store for mapped datatype or data instance.
name := strings.Trim(string(dataspec), "\"")
parts := strings.Split(name, ":")
switch len(parts) {
case 1:
manager.datatypeStore[dvid.TypeString(name)] = store
case 2:
dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
manager.instanceStore[dataid] = store
default:
err = fmt.Errorf("bad backend data specification: %s", dataspec)
return
}
}
manager.setup = true
// Setup the graph store
var store dvid.Store
store, err = assignedStoreByType("labelgraph")
if err != nil {
return
}
var ok bool
kvdb, ok := store.(OrderedKeyValueDB)
if !ok {
return false, fmt.Errorf("assigned labelgraph store %q isn't ordered kv db", store)
}
manager.graphDB, err = NewGraphStore(kvdb)
if err != nil {
return false, err
}
manager.graphSetter, ok = manager.graphDB.(GraphSetter)
if !ok {
return false, fmt.Errorf("Database %q cannot support a graph setter", kvdb)
}
manager.graphGetter, ok = manager.graphDB.(GraphGetter)
if !ok {
return false, fmt.Errorf("Database %q cannot support a graph getter", kvdb)
}
return
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
default:
server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(ROI)
attenuationStr := queryValues.Get("attenuation")
if len(attenuationStr) != 0 {
attenuation, err := strconv.Atoi(attenuationStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if attenuation < 1 || attenuation > 7 {
server.BadRequest(w, r, "Attenuation should be from 1 to 7 (divides by 2^n)")
return
}
roiptr.attenuation = uint8(attenuation)
}
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "put" {
fmt.Printf("Setting configuration of data '%s'\n", d.DataName())
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, d.Help())
return
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
return
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
return
case "rawkey":
// GET <api URL>/node/<UUID>/<data name>/rawkey?x=<block x>&y=<block y>&z=<block z>
if len(parts) != 4 {
server.BadRequest(w, r, "rawkey endpoint should be followed by query strings (x, y, and z) giving block coord")
return
}
case "blocks":
// GET <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
// POST <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "%q must be followed by block-coord/span-x", parts[3])
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
data, err := d.GetBlocks(ctx.VersionID(), blockCoord, int32(span))
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
} else {
if err := d.PutBlocks(ctx.VersionID(), blockCoord, span, r.Body); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: Blocks (%s)", r.Method, r.URL)
case "arb":
// GET <api URL>/node/<UUID>/<data name>/arb/<top left>/<top right>/<bottom left>/<res>[/<format>]
if len(parts) < 8 {
server.BadRequest(w, r, "%q must be followed by top-left/top-right/bottom-left/res", parts[3])
return
}
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
img, err := d.GetArbitraryImage(ctx, parts[4], parts[5], parts[6], parts[7])
if err != nil {
server.BadRequest(w, r, err)
return
}
var formatStr string
if len(parts) >= 9 {
formatStr = parts[8]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: Arbitrary image (%s)", r.Method, r.URL)
case "raw", "isotropic":
// GET <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>]
if len(parts) < 7 {
server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
if err != nil {
server.BadRequest(w, r, err)
return
}
vox, err := d.NewVoxels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), vox, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
if queryStrings.Get("throttle") == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
vox, err := d.NewVoxels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), vox, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
} else {
if isotropic {
err := fmt.Errorf("can only PUT 'raw' not 'isotropic' images")
server.BadRequest(w, r, err)
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store voxels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
vox, err := d.NewVoxels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), vox, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
}
default:
server.BadRequest(w, r, "Unrecognized API call for voxels %q. See API help.", d.DataName())
}
}
// switchboard for remote command execution
func handleCommand(cmd *datastore.Request) (reply *datastore.Response, err error) {
if cmd.Name() == "" {
err = fmt.Errorf("Server error: got empty command!")
return
}
reply = new(datastore.Response)
switch cmd.Name() {
case "help":
reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress())
case "shutdown":
dvid.Infof("DVID server halting due to 'shutdown' command.")
reply.Text = fmt.Sprintf("DVID server at %s is being shutdown...\n", config.RPCAddress())
// launch goroutine shutdown so we can concurrently return shutdown message to client.
go Shutdown()
case "types":
if len(cmd.Command) == 1 {
text := "\nData Types within this DVID Server\n"
text += "----------------------------------\n"
var mapTypes map[dvid.URLString]datastore.TypeService
if mapTypes, err = datastore.Types(); err != nil {
err = fmt.Errorf("Error trying to retrieve data types within this DVID server!")
return
}
for url, typeservice := range mapTypes {
text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url)
}
reply.Text = text
} else {
if len(cmd.Command) != 3 || cmd.Command[2] != "help" {
err = fmt.Errorf("Unknown types command: %q", cmd.Command)
return
}
var typename string
var typeservice datastore.TypeService
cmd.CommandArgs(1, &typename)
if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil {
return
}
reply.Text = typeservice.Help()
}
case "repos":
var subcommand string
cmd.CommandArgs(1, &subcommand)
switch subcommand {
case "new":
var alias, description string
cmd.CommandArgs(2, &alias, &description)
config := cmd.Settings()
var uuidStr, passcode string
var found bool
if uuidStr, found, err = config.GetString("uuid"); err != nil {
return
}
var assign *dvid.UUID
if !found {
assign = nil
} else {
uuid := dvid.UUID(uuidStr)
assign = &uuid
}
if passcode, found, err = config.GetString("passcode"); err != nil {
return
}
var root dvid.UUID
root, err = datastore.NewRepo(alias, description, assign, passcode)
if err != nil {
return
}
if err = datastore.SetRepoAlias(root, alias); err != nil {
return
}
if err = datastore.SetRepoDescription(root, description); err != nil {
return
}
reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root)
case "delete":
var uuidStr, passcode string
cmd.CommandArgs(2, &uuidStr, &passcode)
var uuid dvid.UUID
if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil {
return
}
if err = datastore.DeleteRepo(uuid, passcode); err != nil {
return
}
reply.Text = fmt.Sprintf("Started deletion of repo %s.\n", uuid)
default:
err = fmt.Errorf("Unknown repos command: %q", subcommand)
return
}
case "repo":
var uuidStr, subcommand string
cmd.CommandArgs(1, &uuidStr, &subcommand)
var uuid dvid.UUID
if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil {
return
}
switch subcommand {
case "new":
var typename, dataname string
cmd.CommandArgs(3, &typename, &dataname)
// Get TypeService
var typeservice datastore.TypeService
if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil {
return
}
// Create new data
config := cmd.Settings()
if _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config); err != nil {
return
}
reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid)
datastore.AddToRepoLog(uuid, []string{cmd.String()})
case "rename":
var name1, name2, passcode string
cmd.CommandArgs(3, &name1, &name2, &passcode)
oldname := dvid.InstanceName(name1)
newname := dvid.InstanceName(name2)
// Make sure this instance exists.
if _, err = datastore.GetDataByUUIDName(uuid, oldname); err != nil {
err = fmt.Errorf("Error trying to rename %q for UUID %s: %v", oldname, uuid, err)
return
}
// Do the rename.
if err = datastore.RenameData(uuid, oldname, newname, passcode); err != nil {
err = fmt.Errorf("Error renaming data instance %q to %q: %v", oldname, newname, err)
return
}
reply.Text = fmt.Sprintf("Renamed data instance %q to %q from DAG subgraph @ root %s\n", oldname, newname, uuid)
case "branch":
cmd.CommandArgs(3, &uuidStr)
var assign *dvid.UUID
if uuidStr == "" {
assign = nil
} else {
u := dvid.UUID(uuidStr)
assign = &u
}
var child dvid.UUID
if child, err = datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign); err != nil {
return
}
reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid)
datastore.AddToRepoLog(uuid, []string{cmd.String()})
case "merge":
uuids := cmd.CommandArgs(2)
parents := make([]dvid.UUID, len(uuids)+1)
parents[0] = dvid.UUID(uuid)
i := 1
for uuid := range uuids {
parents[i] = dvid.UUID(uuid)
i++
}
var child dvid.UUID
child, err = datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree)
if err != nil {
return
}
reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child)
datastore.AddToRepoLog(uuid, []string{cmd.String()})
case "migrate":
var source, oldStoreName string
cmd.CommandArgs(3, &source, &oldStoreName)
var store dvid.Store
store, err = storage.GetStoreByAlias(storage.Alias(oldStoreName))
if err != nil {
return
}
config := cmd.Settings()
go func() {
if err = datastore.MigrateInstance(uuid, dvid.InstanceName(source), store, config); err != nil {
dvid.Errorf("migrate error: %v\n", err)
}
}()
reply.Text = fmt.Sprintf("Started migration of uuid %s data instance %q from old store %q...\n", uuid, source, oldStoreName)
case "copy":
var source, target string
cmd.CommandArgs(3, &source, &target)
config := cmd.Settings()
go func() {
if err = datastore.CopyInstance(uuid, dvid.InstanceName(source), dvid.InstanceName(target), config); err != nil {
dvid.Errorf("copy error: %v\n", err)
}
}()
reply.Text = fmt.Sprintf("Started copy of uuid %s data instance %q to %q...\n", uuid, source, target)
case "push":
var target string
cmd.CommandArgs(3, &target)
config := cmd.Settings()
go func() {
if err = datastore.PushRepo(uuid, target, config); err != nil {
dvid.Errorf("push error: %v\n", err)
}
}()
reply.Text = fmt.Sprintf("Started push of repo %s to %q...\n", uuid, target)
/*
case "pull":
var target string
cmd.CommandArgs(3, &target)
config := cmd.Settings()
if err = datastore.Pull(uuid, target, config); err != nil {
return
}
reply.Text = fmt.Sprintf("Repo %s pulled from %q\n", uuid, target)
*/
case "delete":
var dataname, passcode string
cmd.CommandArgs(3, &dataname, &passcode)
// Make sure this instance exists.
if _, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(dataname)); err != nil {
err = fmt.Errorf("Error trying to delete %q for UUID %s: %v", dataname, uuid, err)
return
}
// Do the deletion. Under hood, modifies metadata immediately and launches async k/v deletion.
if err = datastore.DeleteDataByName(uuid, dvid.InstanceName(dataname), passcode); err != nil {
err = fmt.Errorf("Error deleting data instance %q: %v", dataname, err)
return
}
reply.Text = fmt.Sprintf("Started deletion of data instance %q from repo with root %s\n", dataname, uuid)
default:
err = fmt.Errorf("Unknown command: %q", cmd)
return
}
case "node":
var uuidStr, descriptor string
cmd.CommandArgs(1, &uuidStr, &descriptor)
var uuid dvid.UUID
if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil {
return
}
// Get the DataService
dataname := dvid.InstanceName(descriptor)
var subcommand string
cmd.CommandArgs(3, &subcommand)
var dataservice datastore.DataService
if dataservice, err = datastore.GetDataByUUIDName(uuid, dataname); err != nil {
return
}
if subcommand == "help" {
reply.Text = dataservice.Help()
return
}
err = dataservice.DoRPC(*cmd, reply)
return
default:
// Check to see if it's a name of a compiled data type, in which case we refer it to the data type.
types := datastore.CompiledTypes()
for name, typeservice := range types {
if name == dvid.TypeString(cmd.Argument(0)) {
err = typeservice.Do(*cmd, reply)
return
}
}
err = fmt.Errorf("Unknown command: '%s'", *cmd)
}
return
}
// CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale
func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error {
timedLog := dvid.NewTimeLog()
// Parse the request
var uuidStr, dataName, cmdStr, grayscaleName, destName string
request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName)
// Get the version
uuid, v, err := datastore.MatchingUUID(uuidStr)
if err != nil {
return err
}
// Log request
if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
return err
}
// Get the grayscale data.
dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(grayscaleName))
if err != nil {
return err
}
grayscale, ok := dataservice.(*imageblk.Data)
if !ok {
return fmt.Errorf("%s is not the name of uint8 data", grayscaleName)
}
// Create a new rgba8blk data.
var compservice datastore.DataService
compservice, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(destName))
if err == nil {
return fmt.Errorf("Data instance with name %q already exists", destName)
}
typeService, err := datastore.TypeServiceByName("rgba8blk")
if err != nil {
return fmt.Errorf("Could not get rgba8 type service from DVID")
}
config := dvid.NewConfig()
compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config)
if err != nil {
return err
}
composite, ok := compservice.(*imageblk.Data)
if !ok {
return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName)
}
// Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time.
wg := new(sync.WaitGroup)
op := &compositeOp{grayscale, composite, v}
chunkOp := &storage.ChunkOp{op, wg}
store, err := d.GetOrderedKeyValueDB()
if err != nil {
return err
}
ctx := datastore.NewVersionedCtx(d, v)
extents := d.Extents()
blockBeg := imageblk.NewTKey(extents.MinIndex)
blockEnd := imageblk.NewTKey(extents.MaxIndex)
err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk))
wg.Wait()
// Set new mapped data to same extents.
composite.Properties.Extents = grayscale.Properties.Extents
if err := datastore.SaveDataByUUID(uuid, composite); err != nil {
dvid.Infof("Could not save new data '%s': %v\n", destName, err)
}
timedLog.Infof("Created composite of %s and %s", grayscaleName, destName)
return nil
}