Esempio n. 1
0
func repoDeleteInstanceHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	uuid := c.Env["uuid"].(dvid.UUID)
	queryValues := r.URL.Query()
	imsure := queryValues.Get("imsure")
	if imsure != "true" {
		BadRequest(w, r, "Cannot delete instance unless query string 'imsure=true' is present!")
		return
	}

	dataname, ok := c.URLParams["dataname"]
	if !ok {
		BadRequest(w, r, "Error retrieving data instance name from URL parameters")
		return
	}

	// Make sure this instance exists.
	_, err := datastore.GetDataByUUID(uuid, dvid.InstanceName(dataname))
	if err != nil {
		BadRequest(w, r, "Error trying to delete %q for UUID %s: %v", dataname, uuid, err)
		return
	}

	// Do the deletion.  Under hood, modifies metadata immediately and launches async k/v deletion.
	if err := datastore.DeleteDataByUUID(uuid, dvid.InstanceName(dataname)); err != nil {
		BadRequest(w, r, "Error deleting data instance %q: %v", dataname, err)
		return
	}

	// Just respond that deletion was successfully started
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"result": "Started deletion of data instance %q from repo with root %s"}`,
		dataname, uuid)
}
Esempio n. 2
0
// instanceSelector retrieves the data instance given its complete string name and
// forwards the request to that instance's HTTP handler.
func instanceSelector(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		var err error
		dataname := dvid.InstanceName(c.URLParams["dataname"])
		uuid, ok := c.Env["uuid"].(dvid.UUID)
		if !ok {
			msg := fmt.Sprintf("Bad format for UUID %q\n", c.Env["uuid"])
			BadRequest(w, r, msg)
			return
		}
		data, err := datastore.GetDataByUUID(uuid, dataname)
		if err != nil {
			BadRequest(w, r, err)
			return
		}
		v, err := datastore.VersionFromUUID(uuid)
		if err != nil {
			BadRequest(w, r, err)
		}
		ctx := datastore.NewVersionedCtx(data, v)

		// Handle DVID-wide query string commands like non-interactive call designations
		queryValues := r.URL.Query()

		// All HTTP requests are interactive so let server tally request.
		interactive := queryValues.Get("interactive")
		if interactive == "" || (interactive != "false" && interactive != "0") {
			GotInteractiveRequest()
		}

		// TODO: setup routing for data instances as well.
		data.ServeHTTP(uuid, ctx, w, r)
	}
	return http.HandlerFunc(fn)
}
Esempio n. 3
0
func repoDeleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	uuid := c.Env["uuid"].(dvid.UUID)
	queryValues := r.URL.Query()
	imsure := queryValues.Get("imsure")
	if imsure != "true" {
		BadRequest(w, r, "Cannot delete instance unless query string 'imsure=true' is present!")
		return
	}

	dataname, ok := c.URLParams["dataname"]
	if !ok {
		BadRequest(w, r, "Error in retrieving data instance name from URL parameters")
		return
	}

	// Do the deletion asynchronously since they can take a very long time.
	go func() {
		if err := datastore.DeleteDataByUUID(uuid, dvid.InstanceName(dataname)); err != nil {
			dvid.Errorf("Error in deleting data instance %q: %v", dataname, err)
		}
	}()

	// Just respond that deletion was successfully started
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"result": "Started deletion of data instance %q from repo with root %s"}`,
		dataname, uuid)
}
Esempio n. 4
0
// DataBySpec returns a ROI Data based on a string specification of the form
// "<roiname>,<uuid>". If the given string is not parsable, the "found" return value is false.
func DataBySpec(spec string) (d *Data, v dvid.VersionID, found bool, err error) {
	roispec := strings.Split(spec, ",")
	if len(roispec) != 2 {
		err = fmt.Errorf("Expect ROI filters to have format %q, but got %q", "roi:<roiname>,<uuid>", spec)
		return
	}
	roiName := dvid.InstanceName(roispec[0])
	_, v, err = datastore.MatchingUUID(roispec[1])
	if err != nil {
		return
	}
	var data datastore.DataService
	data, err = datastore.GetDataByVersionName(v, roiName)
	if err != nil {
		return
	}
	var ok bool
	d, ok = data.(*Data)
	if !ok {
		err = fmt.Errorf("Data instance %q is not ROI instance", roiName)
		return
	}
	found = true
	return
}
Esempio n. 5
0
func repoNewDataHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	uuid := c.Env["uuid"].(dvid.UUID)
	config := dvid.NewConfig()
	if err := config.SetByJSON(r.Body); err != nil {
		BadRequest(w, r, fmt.Sprintf("Error decoding POSTed JSON config for 'new': %v", err))
		return
	}

	// Make sure that the passed configuration has data type and instance name.
	typename, found, err := config.GetString("typename")
	if !found || err != nil {
		BadRequest(w, r, "POST on repo endpoint requires specification of valid 'typename'")
		return
	}
	dataname, found, err := config.GetString("dataname")
	if !found || err != nil {
		BadRequest(w, r, "POST on repo endpoint requires specification of valid 'dataname'")
		return
	}
	typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename))
	if err != nil {
		BadRequest(w, r, err)
		return
	}
	_, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config)
	if err != nil {
		BadRequest(w, r, err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{%q: 'Added %s [%s] to node %s'}", "result", dataname, typename, uuid)
}
Esempio n. 6
0
// ForegroundROI creates a new ROI by determining all non-background blocks.
func (d *Data) ForegroundROI(req datastore.Request, reply *datastore.Response) error {
	if d.Values.BytesPerElement() != 1 {
		return fmt.Errorf("Foreground ROI command only implemented for 1 byte/voxel data!")
	}

	// Parse the request
	var uuidStr, dataName, cmdStr, destName, backgroundStr string
	req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &destName, &backgroundStr)

	// Get the version and repo
	uuid, versionID, err := datastore.MatchingUUID(uuidStr)
	if err != nil {
		return err
	}
	if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil {
		return err
	}

	// Use existing destination data or a new ROI data.
	var dest *roi.Data
	dest, err = roi.GetByUUID(uuid, dvid.InstanceName(destName))
	if err != nil {
		config := dvid.NewConfig()
		typeservice, err := datastore.TypeServiceByName("roi")
		if err != nil {
			return err
		}
		dataservice, err := datastore.NewData(uuid, typeservice, dvid.InstanceName(destName), config)
		if err != nil {
			return err
		}
		var ok bool
		dest, ok = dataservice.(*roi.Data)
		if !ok {
			return fmt.Errorf("Could not create ROI data instance")
		}
	}

	// Asynchronously process the voxels.
	background, err := dvid.StringToPointNd(backgroundStr, ",")
	if err != nil {
		return err
	}
	go d.foregroundROI(versionID, dest, background)

	return nil
}
Esempio n. 7
0
func GetTestDataContext(uuid dvid.UUID, name string, instanceID dvid.InstanceID) *DataContext {
	versionID, found := testUUIDToVersion[uuid]
	if !found {
		return nil
	}
	data := &testData{uuid, dvid.InstanceName(name), instanceID}
	return NewDataContext(data, versionID)
}
Esempio n. 8
0
// ServeTile returns a tile with appropriate Content-Type set.
func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter,
	r *http.Request, parts []string) error {

	if d.Levels == nil || len(d.Levels) == 0 {
		return ErrNoMetadataSet
	}
	tileReq, err := d.parseTileReq(r, parts)

	queryValues := r.URL.Query()
	noblanksStr := dvid.InstanceName(queryValues.Get("noblanks"))
	var noblanks bool
	if noblanksStr == "true" {
		noblanks = true
	}
	var formatStr string
	if len(parts) >= 8 {
		formatStr = parts[7]
	}

	data, err := d.getTileData(ctx, tileReq)
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}
	if data == nil {
		if noblanks {
			http.NotFound(w, r)
			return nil
		}
		img, err := d.getBlankTileImage(uuid, tileReq)
		if err != nil {
			return err
		}
		return dvid.WriteImageHttp(w, img, formatStr)
	}

	switch d.Encoding {
	case LZ4:
		var img dvid.Image
		if err := img.Deserialize(data); err != nil {
			return err
		}
		data, err = img.GetPNG()
		w.Header().Set("Content-type", "image/png")
	case PNG:
		w.Header().Set("Content-type", "image/png")
	case JPG:
		w.Header().Set("Content-type", "image/jpeg")
	}
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}
	if _, err = w.Write(data); err != nil {
		return err
	}
	return nil
}
Esempio n. 9
0
func makeGrayscale(uuid dvid.UUID, t *testing.T, name string) *Data {
	config := dvid.NewConfig()
	dataservice, err := datastore.NewData(uuid, grayscaleT, dvid.InstanceName(name), config)
	if err != nil {
		t.Errorf("Unable to create grayscale instance %q: %v\n", name, err)
	}
	grayscale, ok := dataservice.(*Data)
	if !ok {
		t.Errorf("Can't cast uint8blk data service into Data\n")
	}
	return grayscale
}
Esempio n. 10
0
func setupGroupcache(config GroupcacheConfig) error {
	if config.GB == 0 {
		return nil
	}
	var cacheBytes int64
	cacheBytes = int64(config.GB) << 30

	pool := groupcache.NewHTTPPool(config.Host)
	if pool != nil {
		dvid.Infof("Initializing groupcache with %d GB at %s...\n", config.GB, config.Host)
		manager.gcache.cache = groupcache.NewGroup("immutable", cacheBytes, groupcache.GetterFunc(
			func(c groupcache.Context, key string, dest groupcache.Sink) error {
				// Use KeyValueDB defined as context.
				gctx, ok := c.(GroupcacheCtx)
				if !ok {
					return fmt.Errorf("bad groupcache context: expected GroupcacheCtx, got %v", c)
				}

				// First four bytes of key is instance ID to isolate groupcache collisions.
				tk := TKey(key[4:])
				data, err := gctx.KeyValueDB.Get(gctx.Context, tk)
				if err != nil {
					return err
				}
				return dest.SetBytes(data)
			}))
		manager.gcache.supported = make(map[dvid.DataSpecifier]struct{})
		for _, dataspec := range config.Instances {
			name := strings.Trim(dataspec, "\"")
			parts := strings.Split(name, ":")
			switch len(parts) {
			case 2:
				dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
				manager.gcache.supported[dataid] = struct{}{}
			default:
				dvid.Errorf("bad data instance specification %q given for groupcache support in config file\n", dataspec)
			}
		}

		// If we have additional peers, add them and start a listener via the HTTP port.
		if len(config.Peers) > 0 {
			peers := []string{config.Host}
			peers = append(peers, config.Peers...)
			pool.Set(peers...)
			dvid.Infof("Groupcache configuration has %d peers in addition to local host.\n", len(config.Peers))
			dvid.Infof("Starting groupcache HTTP server on %s\n", config.Host)
			http.ListenAndServe(config.Host, http.HandlerFunc(pool.ServeHTTP))
		}
	}
	return nil
}
Esempio n. 11
0
// ParseFilterSpec returns the specified ROI instance name and version within a FilterSpec.
// Currently, only one ROI can be specified in a FilterSpec.  Multiple ROIs should use a
// different FilterSpec like "intersect" instead of "roi".
func ParseFilterSpec(spec storage.FilterSpec) (name dvid.InstanceName, v dvid.VersionID, found bool, err error) {
	var filterval string
	filterval, found = spec.GetFilterSpec("roi")
	if !found {
		return
	}
	roispec := strings.Split(filterval, ",")
	if len(roispec) != 2 {
		err = fmt.Errorf("bad ROI spec: %s", filterval)
		return
	}
	name = dvid.InstanceName(roispec[0])
	_, v, err = datastore.MatchingUUID(roispec[1])
	return
}
Esempio n. 12
0
// Make a copy of a repository, customizing it via config.
// TODO -- modify data instance properties based on filters.
func (r *repoT) customize(v dvid.VersionID, config dvid.Config) (*repoT, rpc.Transmit, error) {
	// Since we can have names separated by commas, split them
	namesStr, found, err := config.GetString("data")
	if err != nil {
		return nil, rpc.TransmitUnknown, err
	}
	var datanames dvid.InstanceNames
	if found {
		for _, name := range strings.Split(namesStr, ",") {
			datanames = append(datanames, dvid.InstanceName(strings.TrimSpace(name)))
		}
	}

	// Check the transmission behavior: all, flatten, or deltas.
	var versions map[dvid.VersionID]struct{}

	transmitStr, found, err := config.GetString("transmit")
	if err != nil {
		return nil, rpc.TransmitUnknown, err
	}
	if !found {
		transmitStr = "all"
	}
	var transmit rpc.Transmit
	switch transmitStr {
	case "flatten":
		transmit = rpc.TransmitFlatten
		versions = map[dvid.VersionID]struct{}{
			v: struct{}{},
		}
	case "all":
		transmit = rpc.TransmitAll
		versions = r.versionSet()
	case "branch":
		transmit = rpc.TransmitBranch
		versions = r.versionSet()
	default:
		return nil, rpc.TransmitUnknown, fmt.Errorf("unknown transmit %s", transmitStr)
	}

	// Make a copy filtering by allowed data instances.
	r.RLock()
	defer r.RUnlock()

	dup, err := r.duplicate(versions, datanames)
	return dup, transmit, err
}
Esempio n. 13
0
// SetSyncByJSON takes a JSON object of sync names and UUID, and creates the sync graph
// and sets the data instance's sync.  If replace is false (default), the new sync
// is appended to the current syncs.
func SetSyncByJSON(d dvid.Data, uuid dvid.UUID, replace bool, in io.ReadCloser) error {
	if manager == nil {
		return ErrManagerNotInitialized
	}
	jsonData := make(map[string]string)
	decoder := json.NewDecoder(in)
	if err := decoder.Decode(&jsonData); err != nil && err != io.EOF {
		return fmt.Errorf("Malformed JSON request in sync request: %v", err)
	}
	syncedCSV, ok := jsonData["sync"]
	if !ok {
		return fmt.Errorf("Could not find 'sync' value in POSTed JSON to sync request.")
	}

	syncedNames := strings.Split(syncedCSV, ",")
	if len(syncedNames) == 0 || (len(syncedNames) == 1 && syncedNames[0] == "") {
		syncedNames = []string{}
	}

	if len(syncedNames) == 0 && !replace {
		dvid.Infof("Ignored attempt to append no syncs to instance %q.\n", d.DataName())
		return nil
	}

	// Make sure all synced names currently exist under this UUID, then transform to data UUIDs.
	syncs := make(dvid.UUIDSet)
	for _, name := range syncedNames {
		data, err := GetDataByUUIDName(uuid, dvid.InstanceName(name))
		if err != nil {
			return err
		}
		syncs[data.DataUUID()] = struct{}{}
	}

	if err := SetSyncData(d, syncs, replace); err != nil {
		return err
	}
	return nil
}
Esempio n. 14
0
// NewData returns a pointer to labelvol data.
func NewData(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
	// Make sure we have a valid labelvol source
	s, found, err := c.GetString("Source")
	if err != nil {
		return nil, err
	}
	if !found {
		return nil, fmt.Errorf("Cannot make labelsz data without valid 'Source' specifying an associated labelvol.")
	}
	srcname := dvid.InstanceName(s)
	if _, err = labelvol.GetByUUID(uuid, srcname); err != nil {
		return nil, err
	}
	c.Set("sync", s) // This will set base data sync list

	// Initialize the Data for this data type
	basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
	if err != nil {
		return nil, err
	}
	return &Data{basedata, srcname}, nil
}
Esempio n. 15
0
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
	// --- Don't time labelgraph ops because they are very small and frequent.
	// --- TODO -- Implement monitoring system that aggregates logged ops instead of
	// ----------- printing out each one.
	// timedLog := dvid.NewTimeLog()

	db, err := storage.GraphStore()
	if err != nil {
		server.BadRequest(w, r, err)
		return
	}

	// make sure transaction log is created
	d.initializeLog()

	// Break URL request into arguments
	url := r.URL.Path[len(server.WebAPIPath):]
	parts := strings.Split(url, "/")
	if len(parts) < 4 {
		server.BadRequest(w, r, "No resource specified in URI")
		return
	}
	method := strings.ToLower(r.Method)
	if method == "put" {
		server.BadRequest(w, r, "PUT requests not supported")
		return
	}

	// Process help and info.
	switch parts[3] {
	case "help":
		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprintln(w, d.Help())
		return
	case "info":
		jsonBytes, err := d.MarshalJSON()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, string(jsonBytes))
	case "subgraph":
		// disable json schema validation (will speedup POST command)
		queryValues := r.URL.Query()
		disableSchemaT := dvid.InstanceName(queryValues.Get("unsafe"))
		disableSchema := false
		if len(disableSchemaT) != 0 && disableSchemaT == "true" {
			disableSchema = true
		}
		labelgraph, err := d.ExtractGraph(r, disableSchema)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		err = d.handleSubgraphBulk(ctx, db, w, labelgraph, method)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
	case "neighbors":
		if method != "get" {
			server.BadRequest(w, r, "Only supports GETs")
			return
		}
		err := d.handleNeighbors(ctx, db, w, parts[4:])
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
	case "merge":
		if method != "post" {
			server.BadRequest(w, r, "Only supports POSTs")
			return
		}
		labelgraph, err := d.ExtractGraph(r, false)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		err = d.handleMerge(ctx, db, w, labelgraph)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
	case "weight":
		if method != "post" {
			server.BadRequest(w, r, "Only supports POSTs")
			return
		}
		labelgraph, err := d.ExtractGraph(r, false)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		err = d.handleWeightUpdate(ctx, db, w, labelgraph)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
	case "propertytransaction":
		err := d.handlePropertyTransaction(ctx, db, w, r, parts[4:], method)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
	case "property":
		err := d.handleProperty(ctx, db, w, r, parts[4:], method)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
	case "undomerge":
		// not supported until transaction history is supported
		server.BadRequest(w, r, "undomerge not yet implemented")
		return
	default:
		server.BadRequest(w, r, "%s not found", parts[3])
		return
	}
	//timedLog.Infof("Successful labelgraph op %s on %q", parts[3], d.DataName())
}
Esempio n. 16
0
// Do acts as a switchboard for remote command execution
func (c *RPCConnection) Do(cmd datastore.Request, reply *datastore.Response) error {
	if reply == nil {
		dvid.Debugf("reply is nil coming in!\n")
		return nil
	}
	if cmd.Name() == "" {
		return fmt.Errorf("Server error: got empty command!")
	}

	switch cmd.Name() {

	case "help":
		reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress())

	case "shutdown":
		Shutdown()
		// Make this process shutdown in a second to allow time for RPC to finish.
		// TODO -- Better way to do this?
		log.Printf("DVID server halted due to 'shutdown' command.")
		reply.Text = fmt.Sprintf("DVID server at %s has been halted.\n", config.RPCAddress())
		go func() {
			time.Sleep(1 * time.Second)
			os.Exit(0)
		}()

	case "types":
		if len(cmd.Command) == 1 {
			text := "\nData Types within this DVID Server\n"
			text += "----------------------------------\n"
			mapTypes, err := datastore.Types()
			if err != nil {
				return fmt.Errorf("Error trying to retrieve data types within this DVID server!")
			}
			for url, typeservice := range mapTypes {
				text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url)
			}
			reply.Text = text
		} else {
			if len(cmd.Command) != 3 || cmd.Command[2] != "help" {
				return fmt.Errorf("Unknown types command: %q", cmd.Command)
			}
			var typename string
			cmd.CommandArgs(1, &typename)
			typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename))
			if err != nil {
				return err
			}
			reply.Text = typeservice.Help()
		}

	case "repos":
		var subcommand, alias, description, uuidStr string
		cmd.CommandArgs(1, &subcommand, &alias, &description, &uuidStr)
		switch subcommand {
		case "new":
			var assign *dvid.UUID
			if uuidStr == "" {
				assign = nil
			} else {
				u := dvid.UUID(uuidStr)
				assign = &u
			}
			root, err := datastore.NewRepo(alias, description, assign)
			if err != nil {
				return err
			}
			if err := datastore.SetRepoAlias(root, alias); err != nil {
				return err
			}
			if err := datastore.SetRepoDescription(root, description); err != nil {
				return err
			}
			reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root)
		default:
			return fmt.Errorf("Unknown repos command: %q", subcommand)
		}

	case "repo":
		var uuidStr, subcommand string
		cmd.CommandArgs(1, &uuidStr, &subcommand)
		uuid, _, err := datastore.MatchingUUID(uuidStr)
		if err != nil {
			return err
		}

		switch subcommand {
		case "new":
			var typename, dataname string
			cmd.CommandArgs(3, &typename, &dataname)

			// Get TypeService
			typeservice, err := datastore.TypeServiceByName(dvid.TypeString(typename))
			if err != nil {
				return err
			}

			// Create new data
			config := cmd.Settings()
			_, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config)
			if err != nil {
				return err
			}
			reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid)
			datastore.AddToRepoLog(uuid, []string{cmd.String()})

		case "branch":
			cmd.CommandArgs(3, &uuidStr)

			var assign *dvid.UUID
			if uuidStr == "" {
				assign = nil
			} else {
				u := dvid.UUID(uuidStr)
				assign = &u
			}
			child, err := datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign)
			if err != nil {
				return err
			}
			reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid)
			datastore.AddToRepoLog(uuid, []string{cmd.String()})

		case "merge":
			uuids := cmd.CommandArgs(2)

			parents := make([]dvid.UUID, len(uuids)+1)
			parents[0] = dvid.UUID(uuid)
			i := 1
			for uuid := range uuids {
				parents[i] = dvid.UUID(uuid)
				i++
			}
			child, err := datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree)
			if err != nil {
				return err
			}
			reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child)
			datastore.AddToRepoLog(uuid, []string{cmd.String()})

		case "push":
			/*
				var target string
				cmd.CommandArgs(3, &target)
				config := cmd.Settings()
				if err = datastore.Push(repo, target, config); err != nil {
					return err
				}
				reply.Text = fmt.Sprintf("Repo %q pushed to %q\n", repo.RootUUID(), target)
			*/
			return fmt.Errorf("push command has been temporarily suspended")
		default:
			return fmt.Errorf("Unknown command: %q", cmd)
		}

	case "node":
		var uuidStr, descriptor string
		cmd.CommandArgs(1, &uuidStr, &descriptor)
		uuid, _, err := datastore.MatchingUUID(uuidStr)
		if err != nil {
			return err
		}

		// Get the DataService
		dataname := dvid.InstanceName(descriptor)
		var subcommand string
		cmd.CommandArgs(3, &subcommand)
		dataservice, err := datastore.GetDataByUUID(uuid, dataname)
		if err != nil {
			return err
		}
		if subcommand == "help" {
			reply.Text = dataservice.Help()
			return nil
		}
		return dataservice.DoRPC(cmd, reply)

	default:
		return fmt.Errorf("Unknown command: '%s'", cmd)
	}
	return nil
}
Esempio n. 17
0
// instanceSelector retrieves the data instance given its complete string name and
// forwards the request to that instance's HTTP handler.
func instanceSelector(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		if httpUnavailable(w) {
			return
		}

		var err error
		dataname := dvid.InstanceName(c.URLParams["dataname"])
		uuid, ok := c.Env["uuid"].(dvid.UUID)
		if !ok {
			msg := fmt.Sprintf("Bad format for UUID %q\n", c.Env["uuid"])
			BadRequest(w, r, msg)
			return
		}
		data, err := datastore.GetDataByUUIDName(uuid, dataname)
		if err != nil {
			BadRequest(w, r, err)
			return
		}
		v, err := datastore.VersionFromUUID(uuid)
		if err != nil {
			BadRequest(w, r, err)
			return
		}

		if data.Versioned() {
			// Make sure we aren't trying mutable methods on committed nodes.
			locked, err := datastore.LockedUUID(uuid)
			if err != nil {
				BadRequest(w, r, err)
				return
			}
			if locked && data.IsMutationRequest(r.Method, c.URLParams["keyword"]) {
				BadRequest(w, r, "Cannot do %s on endpoint %q of locked node %s", r.Method, c.URLParams["keyword"], uuid)
				return
			}
		} else {
			// Map everything to root version.
			v, err = datastore.GetRepoRootVersion(v)
			if err != nil {
				BadRequest(w, r, err)
				return
			}
		}
		ctx := datastore.NewVersionedCtx(data, v)

		// Also set the web request information in case logging needs it downstream.
		ctx.SetRequestID(middleware.GetReqID(*c))

		// Handle DVID-wide query string commands like non-interactive call designations
		queryStrings := r.URL.Query()

		// All HTTP requests are interactive so let server tally request.
		interactive := queryStrings.Get("interactive")
		if interactive == "" || (interactive != "false" && interactive != "0") {
			GotInteractiveRequest()
		}

		// TODO: setup routing for data instances as well.
		if config != nil && config.AllowTiming() {
			w.Header().Set("Timing-Allow-Origin", "*")
		}
		data.ServeHTTP(uuid, ctx, w, r)
	}
	return http.HandlerFunc(fn)
}
Esempio n. 18
0
// handleTileReq returns a tile with appropriate Content-Type set.
func (d *Data) handleTileReq(w http.ResponseWriter, r *http.Request, parts []string) error {

	if len(parts) < 7 {
		return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
	}
	planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
	queryStrings := r.URL.Query()

	var noblanks bool
	noblanksStr := dvid.InstanceName(queryStrings.Get("noblanks"))
	if noblanksStr == "true" {
		noblanks = true
	}

	var tilesize int32 = DefaultTileSize
	tileSizeStr := queryStrings.Get("tilesize")
	if tileSizeStr != "" {
		tilesizeInt, err := strconv.Atoi(tileSizeStr)
		if err != nil {
			return err
		}
		tilesize = int32(tilesizeInt)
	}
	size := dvid.Point2d{tilesize, tilesize}

	var formatStr string
	if len(parts) >= 8 {
		formatStr = parts[7]
	}
	if formatStr == "" {
		formatStr = DefaultTileFormat
	}

	// Parse the tile specification
	plane := dvid.DataShapeString(planeStr)
	shape, err := plane.DataShape()
	if err != nil {
		err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	scale, err := strconv.ParseUint(scalingStr, 10, 8)
	if err != nil {
		err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	tileCoord, err := dvid.StringToPoint(coordStr, "_")
	if err != nil {
		err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
		server.BadRequest(w, r, err)
		return err
	}

	// Convert tile coordinate to offset.
	var ox, oy, oz int32
	switch {
	case shape.Equals(dvid.XY):
		ox = tileCoord.Value(0) * tilesize
		oy = tileCoord.Value(1) * tilesize
		oz = tileCoord.Value(2)
	case shape.Equals(dvid.XZ):
		ox = tileCoord.Value(0) * tilesize
		oy = tileCoord.Value(1)
		oz = tileCoord.Value(2) * tilesize
	case shape.Equals(dvid.YZ):
		ox = tileCoord.Value(0)
		oy = tileCoord.Value(1) * tilesize
		oz = tileCoord.Value(2) * tilesize
	default:
		return fmt.Errorf("Unknown tile orientation: %s", shape)
	}

	// Determine how this request sits in the available scaled volumes.
	geom, err := d.GetGoogleSubvolGeom(Scaling(scale), shape, dvid.Point3d{ox, oy, oz}, size)
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}

	// Send the tile.
	return d.serveTile(w, r, geom, formatStr, noblanks)
}
Esempio n. 19
0
// ServeTile returns a tile with appropriate Content-Type set.
func (d *Data) ServeTile(uuid dvid.UUID, ctx storage.Context, w http.ResponseWriter,
	r *http.Request, parts []string) error {

	if len(parts) < 7 {
		return fmt.Errorf("'tile' request must be following by plane, scale level, and tile coordinate")
	}
	planeStr, scalingStr, coordStr := parts[4], parts[5], parts[6]
	queryValues := r.URL.Query()
	noblanksStr := dvid.InstanceName(queryValues.Get("noblanks"))
	var noblanks bool
	if noblanksStr == "true" {
		noblanks = true
	}
	var formatStr string
	if len(parts) >= 8 {
		formatStr = parts[7]
	}

	// Construct the index for this tile
	plane := dvid.DataShapeString(planeStr)
	shape, err := plane.DataShape()
	if err != nil {
		err = fmt.Errorf("Illegal tile plane: %s (%v)", planeStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	scaling, err := strconv.ParseUint(scalingStr, 10, 8)
	if err != nil {
		err = fmt.Errorf("Illegal tile scale: %s (%v)", scalingStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	tileCoord, err := dvid.StringToPoint(coordStr, "_")
	if err != nil {
		err = fmt.Errorf("Illegal tile coordinate: %s (%v)", coordStr, err)
		server.BadRequest(w, r, err)
		return err
	}
	if tileCoord.NumDims() != 3 {
		err = fmt.Errorf("Expected 3d tile coordinate, got %s", coordStr)
		server.BadRequest(w, r, err)
		return err
	}
	indexZYX := dvid.IndexZYX{tileCoord.Value(0), tileCoord.Value(1), tileCoord.Value(2)}
	data, err := d.getTileData(ctx, shape, Scaling(scaling), indexZYX)
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}
	if data == nil {
		if noblanks {
			http.NotFound(w, r)
			return nil
		}
		img, err := d.getBlankTileImage(uuid, shape, Scaling(scaling))
		if err != nil {
			return err
		}
		return dvid.WriteImageHttp(w, img, formatStr)
	}

	switch d.Encoding {
	case LZ4:
		var img dvid.Image
		if err := img.Deserialize(data); err != nil {
			return err
		}
		data, err = img.GetPNG()
		w.Header().Set("Content-type", "image/png")
	case PNG:
		w.Header().Set("Content-type", "image/png")
	case JPG:
		w.Header().Set("Content-type", "image/jpeg")
	}
	if err != nil {
		server.BadRequest(w, r, err)
		return err
	}
	if _, err = w.Write(data); err != nil {
		return err
	}
	return nil
}
Esempio n. 20
0
// NewData returns a pointer to new tile data with default values.
func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) {
	// Make sure we have a valid DataService source
	sourcename, found, err := c.GetString("Source")
	if err != nil {
		return nil, err
	}
	if !found {
		return nil, fmt.Errorf("Cannot make imagetile data without valid 'Source' setting.")
	}

	// See if we want placeholder imagetile.
	placeholder, found, err := c.GetBool("Placeholder")
	if err != nil {
		return nil, err
	}

	// Determine encoding for tile storage and this dictates what kind of compression we use.
	encoding, found, err := c.GetString("Format")
	if err != nil {
		return nil, err
	}
	format := PNG
	if found {
		switch strings.ToLower(encoding) {
		case "lz4":
			format = LZ4
		case "png":
			format = PNG
		case "jpg":
			format = JPG
		default:
			return nil, fmt.Errorf("Unknown encoding specified: '%s' (should be 'lz4', 'png', or 'jpg'", encoding)
		}
	}

	// Compression is determined by encoding.  Inform user if there's a discrepancy.
	var compression string
	switch format {
	case LZ4:
		compression = "lz4"
	case PNG:
		compression = "none"
	case JPG:
		compression = "none"
	}
	compressConfig, found, err := c.GetString("Compression")
	if err != nil {
		return nil, err
	}
	if found && strings.ToLower(compressConfig) != compression {
		return nil, fmt.Errorf("Conflict between specified compression '%s' and format '%s'.  Suggest not dictating compression.",
			compressConfig, encoding)
	}
	c.Set("Compression", compression)

	// Initialize the imagetile data
	basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
	if err != nil {
		return nil, err
	}
	data := &Data{
		Data: basedata,
		Properties: Properties{
			Source:      dvid.InstanceName(sourcename),
			Placeholder: placeholder,
			Encoding:    format,
		},
	}
	return data, nil
}
Esempio n. 21
0
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
	// TODO -- Refactor this method to break it up and make it simpler.  Use the web routing for the endpoints.

	timedLog := dvid.NewTimeLog()

	// Get the action (GET, POST)
	action := strings.ToLower(r.Method)
	switch action {
	case "get":
	case "post":
	case "delete":
	default:
		server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
		return
	}

	// Break URL request into arguments
	url := r.URL.Path[len(server.WebAPIPath):]
	parts := strings.Split(url, "/")
	if len(parts[len(parts)-1]) == 0 {
		parts = parts[:len(parts)-1]
	}

	// Get query strings and possible roi
	var roiptr *imageblk.ROI
	queryValues := r.URL.Query()
	roiname := dvid.InstanceName(queryValues.Get("roi"))
	if len(roiname) != 0 {
		roiptr = new(imageblk.ROI)
	}

	// Handle POST on data -> setting of configuration
	if len(parts) == 3 && action == "post" {
		config, err := server.DecodeJSON(r)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := d.ModifyConfig(config); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := datastore.SaveDataByUUID(uuid, d); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
		return
	}

	if len(parts) < 4 {
		server.BadRequest(w, r, "Incomplete API request")
		return
	}

	// Process help and info.
	switch parts[3] {
	case "help":
		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprintln(w, dtype.Help())

	case "metadata":
		jsonStr, err := d.NdDataMetadata()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
		fmt.Fprintln(w, jsonStr)

	case "info":
		jsonBytes, err := d.MarshalJSON()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, string(jsonBytes))

	case "label":
		// GET <api URL>/node/<UUID>/<data name>/label/<coord>
		if len(parts) < 5 {
			server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
			return
		}
		coord, err := dvid.StringToPoint(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-type", "application/json")
		jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
		fmt.Fprintf(w, jsonStr)
		timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)

	case "labels":
		// POST <api URL>/node/<UUID>/<data name>/labels
		if action != "post" {
			server.BadRequest(w, r, "Batch labels query must be a POST request")
			return
		}
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			server.BadRequest(w, r, "Bad POSTed data for batch reverse query.  Should be JSON.")
			return
		}
		var coords []dvid.Point3d
		if err := json.Unmarshal(data, &coords); err != nil {
			server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
			return
		}
		w.Header().Set("Content-type", "application/json")
		fmt.Fprintf(w, "[")
		sep := false
		for _, coord := range coords {
			label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if sep {
				fmt.Fprintf(w, ",")
			}
			fmt.Fprintf(w, "%d", label)
			sep = true
		}
		fmt.Fprintf(w, "]")
		timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))

	case "blocks":
		// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
		if len(parts) < 6 {
			server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
			return
		}
		blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		span, err := strconv.Atoi(parts[5])
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if action != "delete" {
			server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
			return
		}
		if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)

	case "pseudocolor":
		if len(parts) < 7 {
			server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
			return
		}
		shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
		planeStr := dvid.DataShapeString(shapeStr)
		plane, err := planeStr.DataShape()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		switch plane.ShapeDimensions() {
		case 2:
			slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action != "get" {
				server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
				return
			}
			lbl, err := d.NewLabels(slice, nil)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if roiptr != nil {
				roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}

			// Convert to pseudocolor
			pseudoColor, err := colorImage(img)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}

			//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
			var formatStr string
			if len(parts) >= 8 {
				formatStr = parts[7]
			}
			err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
		default:
			server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
			return
		}

	case "raw", "isotropic":
		if len(parts) < 7 {
			server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
			return
		}
		var isotropic bool = (parts[3] == "isotropic")
		shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
		planeStr := dvid.DataShapeString(shapeStr)
		plane, err := planeStr.DataShape()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		switch plane.ShapeDimensions() {
		case 2:
			slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action != "get" {
				server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
				return
			}
			rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
			lbl, err := d.NewLabels(rawSlice, nil)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if roiptr != nil {
				roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if isotropic {
				dstW := int(slice.Size().Value(0))
				dstH := int(slice.Size().Value(1))
				img, err = img.ScaleImage(dstW, dstH)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			var formatStr string
			if len(parts) >= 8 {
				formatStr = parts[7]
			}
			//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
			err = dvid.WriteImageHttp(w, img.Get(), formatStr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
		case 3:
			queryStrings := r.URL.Query()
			throttle := queryStrings.Get("throttle")
			if throttle == "true" || throttle == "on" {
				select {
				case <-server.Throttle:
					// Proceed with operation, returning throttle token to server at end.
					defer func() {
						server.Throttle <- 1
					}()
				default:
					throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
						server.MaxThrottledOps)
					http.Error(w, throttleMsg, http.StatusServiceUnavailable)
					return
				}
			}
			compression := queryValues.Get("compression")
			subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action == "get" {
				lbl, err := d.NewLabels(subvol, nil)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				if roiptr != nil {
					roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				}
				data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				w.Header().Set("Content-type", "application/octet-stream")
				switch compression {
				case "":
					_, err = w.Write(data)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				case "lz4":
					compressed := make([]byte, lz4.CompressBound(data))
					var n, outSize int
					if outSize, err = lz4.Compress(data, compressed); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					compressed = compressed[:outSize]
					if n, err = w.Write(compressed); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if n != outSize {
						errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
						dvid.Errorf(errmsg)
						server.BadRequest(w, r, errmsg)
						return
					}
				case "gzip":
					gw := gzip.NewWriter(w)
					if _, err = gw.Write(data); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if err = gw.Close(); err != nil {
						server.BadRequest(w, r, err)
						return
					}
				default:
					server.BadRequest(w, r, "unknown compression type %q", compression)
					return
				}
			} else {
				if isotropic {
					server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
					return
				}
				// Make sure vox is block-aligned
				if !dvid.BlockAligned(subvol, d.BlockSize()) {
					server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
					return
				}

				var data []byte
				switch compression {
				case "":
					tlog := dvid.NewTimeLog()
					data, err = ioutil.ReadAll(r.Body)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					tlog.Debugf("read 3d uncompressed POST")
				case "lz4":
					tlog := dvid.NewTimeLog()
					data, err = ioutil.ReadAll(r.Body)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if len(data) == 0 {
						server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
						return
					}
					tlog.Debugf("read 3d lz4 POST")
					tlog = dvid.NewTimeLog()
					uncompressed := make([]byte, subvol.NumVoxels()*8)
					err = lz4.Uncompress(data, uncompressed)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					data = uncompressed
					tlog.Debugf("uncompressed 3d lz4 POST")
				case "gzip":
					tlog := dvid.NewTimeLog()
					gr, err := gzip.NewReader(r.Body)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					data, err = ioutil.ReadAll(gr)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
					if err = gr.Close(); err != nil {
						server.BadRequest(w, r, err)
						return
					}
					tlog.Debugf("read and uncompress 3d gzip POST")
				default:
					server.BadRequest(w, r, "unknown compression type %q", compression)
					return
				}
				lbl, err := d.NewLabels(subvol, data)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				if roiptr != nil {
					roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				}
				if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
		default:
			server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
			return
		}

	default:
		server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q.  See API help.",
			parts[3], d.DataName())
	}
}
Esempio n. 22
0
func (d *Data) ModifyConfig(config dvid.Config) error {
	// Set compression for this instance
	s, found, err := config.GetString("Compression")
	if err != nil {
		return err
	}
	if found {
		format := strings.ToLower(s)
		switch format {
		case "none":
			d.compression, _ = dvid.NewCompression(dvid.Uncompressed, dvid.DefaultCompression)
		case "snappy":
			d.compression, _ = dvid.NewCompression(dvid.Snappy, dvid.DefaultCompression)
		case "lz4":
			d.compression, _ = dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
		case "gzip":
			d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.DefaultCompression)
		default:
			// Check for gzip + compression level
			parts := strings.Split(format, ":")
			if len(parts) == 2 && parts[0] == "gzip" {
				level, err := strconv.Atoi(parts[1])
				if err != nil {
					return fmt.Errorf("Unable to parse gzip compression level ('%d').  Should be 'gzip:<level>'.", parts[1])
				}
				d.compression, _ = dvid.NewCompression(dvid.Gzip, dvid.CompressionLevel(level))
			} else {
				return fmt.Errorf("Illegal compression specified: %s", s)
			}
		}
	}

	// Set checksum for this instance
	s, found, err = config.GetString("Checksum")
	if err != nil {
		return err
	}
	if found {
		checksum := strings.ToLower(s)
		switch checksum {
		case "none":
			d.checksum = dvid.NoChecksum
		case "crc32":
			d.checksum = dvid.CRC32
		default:
			return fmt.Errorf("Illegal checksum specified: %s", s)
		}
	}

	// Set data instances for syncing.
	s, found, err = config.GetString("sync")
	if err != nil {
		return err
	}
	if found {
		names := strings.Split(s, ",")
		if len(names) > 0 {
			for _, name := range names {
				d.syncs = append(d.syncs, dvid.InstanceName(name))
			}
		}
	}

	// Set versioning
	s, found, err = config.GetString("Versioned")
	if err != nil {
		return err
	}
	if found {
		versioned := strings.ToLower(s)
		switch versioned {
		case "false", "0":
			d.unversioned = true
		case "true", "1":
			d.unversioned = false
		default:
			return fmt.Errorf("Illegal setting for 'versioned' (needs to be 'false', '0', 'true', or '1'): %s", s)
		}
	}
	return nil
}
Esempio n. 23
0
// Initialize the storage systems.  Returns a bool + error where the bool is
// true if the metadata store is newly created and needs initialization.
// The map of store configurations should be keyed by either a datatype name,
// "default", or "metadata".
func Initialize(cmdline dvid.Config, backend *Backend) (createdMetadata bool, err error) {
	dvid.Infof("backend:\n%v\n", *backend)
	// Open all the backend stores
	manager.stores = make(map[Alias]dvid.Store, len(backend.Stores))
	var gotDefault, gotMetadata, createdDefault, lastCreated bool
	var lastStore dvid.Store
	for alias, dbconfig := range backend.Stores {
		var store dvid.Store
		for dbalias, db := range manager.stores {
			if db.Equal(dbconfig) {
				return false, fmt.Errorf("Store %q configuration is duplicate of store %q", alias, dbalias)
			}
		}
		store, created, err := NewStore(dbconfig)
		if err != nil {
			return false, fmt.Errorf("bad store %q: %v", alias, err)
		}
		if alias == backend.Metadata {
			gotMetadata = true
			createdMetadata = created
			manager.metadataStore = store
		}
		if alias == backend.Default {
			gotDefault = true
			createdDefault = created
			manager.defaultStore = store
		}
		manager.stores[alias] = store
		lastStore = store
		lastCreated = created
	}

	// Return if we don't have default or metadata stores.  Should really be caught
	// at configuration loading, but here as well as double check.
	if !gotDefault {
		if len(backend.Stores) == 1 {
			manager.defaultStore = lastStore
			createdDefault = lastCreated
		} else {
			return false, fmt.Errorf("either backend.default or a single store must be set in configuration TOML file")
		}
	}
	if !gotMetadata {
		manager.metadataStore = manager.defaultStore
		createdMetadata = createdDefault
	}
	dvid.Infof("Default store: %s\n", manager.defaultStore)
	dvid.Infof("Metadata store: %s\n", manager.metadataStore)

	// Setup the groupcache if specified.
	err = setupGroupcache(backend.Groupcache)
	if err != nil {
		return
	}

	// Make all data instance or datatype-specific store assignments.
	manager.instanceStore = make(map[dvid.DataSpecifier]dvid.Store)
	manager.datatypeStore = make(map[dvid.TypeString]dvid.Store)
	for dataspec, alias := range backend.Mapping {
		if dataspec == "default" || dataspec == "metadata" {
			continue
		}
		store, found := manager.stores[alias]
		if !found {
			err = fmt.Errorf("bad backend store alias: %q -> %q", dataspec, alias)
			return
		}
		// Cache the store for mapped datatype or data instance.
		name := strings.Trim(string(dataspec), "\"")
		parts := strings.Split(name, ":")
		switch len(parts) {
		case 1:
			manager.datatypeStore[dvid.TypeString(name)] = store
		case 2:
			dataid := dvid.GetDataSpecifier(dvid.InstanceName(parts[0]), dvid.UUID(parts[1]))
			manager.instanceStore[dataid] = store
		default:
			err = fmt.Errorf("bad backend data specification: %s", dataspec)
			return
		}
	}
	manager.setup = true

	// Setup the graph store
	var store dvid.Store
	store, err = assignedStoreByType("labelgraph")
	if err != nil {
		return
	}
	var ok bool
	kvdb, ok := store.(OrderedKeyValueDB)
	if !ok {
		return false, fmt.Errorf("assigned labelgraph store %q isn't ordered kv db", store)
	}
	manager.graphDB, err = NewGraphStore(kvdb)
	if err != nil {
		return false, err
	}
	manager.graphSetter, ok = manager.graphDB.(GraphSetter)
	if !ok {
		return false, fmt.Errorf("Database %q cannot support a graph setter", kvdb)
	}
	manager.graphGetter, ok = manager.graphDB.(GraphGetter)
	if !ok {
		return false, fmt.Errorf("Database %q cannot support a graph getter", kvdb)
	}
	return
}
Esempio n. 24
0
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
	timedLog := dvid.NewTimeLog()

	// Get the action (GET, POST)
	action := strings.ToLower(r.Method)
	switch action {
	case "get":
	case "post":
	default:
		server.BadRequest(w, r, "Can only handle GET or POST HTTP verbs")
		return
	}

	// Break URL request into arguments
	url := r.URL.Path[len(server.WebAPIPath):]
	parts := strings.Split(url, "/")
	if len(parts[len(parts)-1]) == 0 {
		parts = parts[:len(parts)-1]
	}

	// Get query strings and possible roi
	var roiptr *ROI
	queryValues := r.URL.Query()
	roiname := dvid.InstanceName(queryValues.Get("roi"))
	if len(roiname) != 0 {
		roiptr = new(ROI)
		attenuationStr := queryValues.Get("attenuation")
		if len(attenuationStr) != 0 {
			attenuation, err := strconv.Atoi(attenuationStr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if attenuation < 1 || attenuation > 7 {
				server.BadRequest(w, r, "Attenuation should be from 1 to 7 (divides by 2^n)")
				return
			}
			roiptr.attenuation = uint8(attenuation)
		}
	}

	// Handle POST on data -> setting of configuration
	if len(parts) == 3 && action == "put" {
		fmt.Printf("Setting configuration of data '%s'\n", d.DataName())
		config, err := server.DecodeJSON(r)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := d.ModifyConfig(config); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if err := datastore.SaveDataByUUID(uuid, d); err != nil {
			server.BadRequest(w, r, err)
			return
		}
		fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
		return
	}

	if len(parts) < 4 {
		server.BadRequest(w, r, "Incomplete API request")
		return
	}

	// Process help and info.
	switch parts[3] {
	case "help":
		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprintln(w, d.Help())
		return

	case "metadata":
		jsonStr, err := d.NdDataMetadata()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
		fmt.Fprintln(w, jsonStr)
		return

	case "info":
		jsonBytes, err := d.MarshalJSON()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, string(jsonBytes))
		return

	case "rawkey":
		// GET <api URL>/node/<UUID>/<data name>/rawkey?x=<block x>&y=<block y>&z=<block z>
		if len(parts) != 4 {
			server.BadRequest(w, r, "rawkey endpoint should be followed by query strings (x, y, and z) giving block coord")
			return
		}

	case "blocks":
		// GET  <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
		// POST <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
		if len(parts) < 6 {
			server.BadRequest(w, r, "%q must be followed by block-coord/span-x", parts[3])
			return
		}
		blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		span, err := strconv.Atoi(parts[5])
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		if action == "get" {
			data, err := d.GetBlocks(ctx.VersionID(), blockCoord, int32(span))
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			w.Header().Set("Content-type", "application/octet-stream")
			_, err = w.Write(data)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
		} else {
			if err := d.PutBlocks(ctx.VersionID(), blockCoord, span, r.Body); err != nil {
				server.BadRequest(w, r, err)
				return
			}
		}
		timedLog.Infof("HTTP %s: Blocks (%s)", r.Method, r.URL)

	case "arb":
		// GET  <api URL>/node/<UUID>/<data name>/arb/<top left>/<top right>/<bottom left>/<res>[/<format>]
		if len(parts) < 8 {
			server.BadRequest(w, r, "%q must be followed by top-left/top-right/bottom-left/res", parts[3])
			return
		}
		queryStrings := r.URL.Query()
		throttle := queryStrings.Get("throttle")
		if throttle == "true" || throttle == "on" {
			select {
			case <-server.Throttle:
				// Proceed with operation, returning throttle token to server at end.
				defer func() {
					server.Throttle <- 1
				}()
			default:
				throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
					server.MaxThrottledOps)
				http.Error(w, throttleMsg, http.StatusServiceUnavailable)
				return
			}
		}
		img, err := d.GetArbitraryImage(ctx, parts[4], parts[5], parts[6], parts[7])
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		var formatStr string
		if len(parts) >= 9 {
			formatStr = parts[8]
		}
		err = dvid.WriteImageHttp(w, img.Get(), formatStr)
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		timedLog.Infof("HTTP %s: Arbitrary image (%s)", r.Method, r.URL)

	case "raw", "isotropic":
		// GET  <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>]
		if len(parts) < 7 {
			server.BadRequest(w, r, "%q must be followed by shape/size/offset", parts[3])
			return
		}
		var isotropic bool = (parts[3] == "isotropic")
		shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
		planeStr := dvid.DataShapeString(shapeStr)
		plane, err := planeStr.DataShape()
		if err != nil {
			server.BadRequest(w, r, err)
			return
		}
		switch plane.ShapeDimensions() {
		case 2:
			slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action != "get" {
				server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
				return
			}
			rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			vox, err := d.NewVoxels(rawSlice, nil)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if roiptr != nil {
				roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			img, err := d.GetImage(ctx.VersionID(), vox, roiptr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if isotropic {
				dstW := int(slice.Size().Value(0))
				dstH := int(slice.Size().Value(1))
				img, err = img.ScaleImage(dstW, dstH)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			var formatStr string
			if len(parts) >= 8 {
				formatStr = parts[7]
			}
			err = dvid.WriteImageHttp(w, img.Get(), formatStr)
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
		case 3:
			queryStrings := r.URL.Query()
			if queryStrings.Get("throttle") == "on" {
				select {
				case <-server.Throttle:
					// Proceed with operation, returning throttle token to server at end.
					defer func() {
						server.Throttle <- 1
					}()
				default:
					throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
						server.MaxThrottledOps)
					http.Error(w, throttleMsg, http.StatusServiceUnavailable)
					return
				}
			}
			subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
			if err != nil {
				server.BadRequest(w, r, err)
				return
			}
			if action == "get" {
				vox, err := d.NewVoxels(subvol, nil)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				if roiptr != nil {
					roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				}
				data, err := d.GetVolume(ctx.VersionID(), vox, roiptr)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				w.Header().Set("Content-type", "application/octet-stream")
				_, err = w.Write(data)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
			} else {
				if isotropic {
					err := fmt.Errorf("can only PUT 'raw' not 'isotropic' images")
					server.BadRequest(w, r, err)
					return
				}
				// Make sure vox is block-aligned
				if !dvid.BlockAligned(subvol, d.BlockSize()) {
					server.BadRequest(w, r, "cannot store voxels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
					return
				}

				data, err := ioutil.ReadAll(r.Body)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				vox, err := d.NewVoxels(subvol, data)
				if err != nil {
					server.BadRequest(w, r, err)
					return
				}
				if roiptr != nil {
					roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), vox)
					if err != nil {
						server.BadRequest(w, r, err)
						return
					}
				}
				if err = d.PutVoxels(ctx.VersionID(), vox, roiptr); err != nil {
					server.BadRequest(w, r, err)
					return
				}
			}
			timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
		default:
			server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
		}
	default:
		server.BadRequest(w, r, "Unrecognized API call for voxels %q.  See API help.", d.DataName())
	}
}
Esempio n. 25
0
// switchboard for remote command execution
func handleCommand(cmd *datastore.Request) (reply *datastore.Response, err error) {
	if cmd.Name() == "" {
		err = fmt.Errorf("Server error: got empty command!")
		return
	}
	reply = new(datastore.Response)

	switch cmd.Name() {

	case "help":
		reply.Text = fmt.Sprintf(RPCHelpMessage, config.RPCAddress(), config.HTTPAddress())

	case "shutdown":
		dvid.Infof("DVID server halting due to 'shutdown' command.")
		reply.Text = fmt.Sprintf("DVID server at %s is being shutdown...\n", config.RPCAddress())
		// launch goroutine shutdown so we can concurrently return shutdown message to client.
		go Shutdown()

	case "types":
		if len(cmd.Command) == 1 {
			text := "\nData Types within this DVID Server\n"
			text += "----------------------------------\n"
			var mapTypes map[dvid.URLString]datastore.TypeService
			if mapTypes, err = datastore.Types(); err != nil {
				err = fmt.Errorf("Error trying to retrieve data types within this DVID server!")
				return
			}
			for url, typeservice := range mapTypes {
				text += fmt.Sprintf("%-20s %s\n", typeservice.GetTypeName(), url)
			}
			reply.Text = text
		} else {
			if len(cmd.Command) != 3 || cmd.Command[2] != "help" {
				err = fmt.Errorf("Unknown types command: %q", cmd.Command)
				return
			}
			var typename string
			var typeservice datastore.TypeService
			cmd.CommandArgs(1, &typename)
			if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil {
				return
			}
			reply.Text = typeservice.Help()
		}

	case "repos":
		var subcommand string
		cmd.CommandArgs(1, &subcommand)

		switch subcommand {
		case "new":
			var alias, description string
			cmd.CommandArgs(2, &alias, &description)

			config := cmd.Settings()
			var uuidStr, passcode string
			var found bool
			if uuidStr, found, err = config.GetString("uuid"); err != nil {
				return
			}
			var assign *dvid.UUID
			if !found {
				assign = nil
			} else {
				uuid := dvid.UUID(uuidStr)
				assign = &uuid
			}
			if passcode, found, err = config.GetString("passcode"); err != nil {
				return
			}
			var root dvid.UUID
			root, err = datastore.NewRepo(alias, description, assign, passcode)
			if err != nil {
				return
			}
			if err = datastore.SetRepoAlias(root, alias); err != nil {
				return
			}
			if err = datastore.SetRepoDescription(root, description); err != nil {
				return
			}
			reply.Text = fmt.Sprintf("New repo %q created with head node %s\n", alias, root)

		case "delete":
			var uuidStr, passcode string
			cmd.CommandArgs(2, &uuidStr, &passcode)

			var uuid dvid.UUID
			if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil {
				return
			}
			if err = datastore.DeleteRepo(uuid, passcode); err != nil {
				return
			}
			reply.Text = fmt.Sprintf("Started deletion of repo %s.\n", uuid)

		default:
			err = fmt.Errorf("Unknown repos command: %q", subcommand)
			return
		}

	case "repo":
		var uuidStr, subcommand string
		cmd.CommandArgs(1, &uuidStr, &subcommand)
		var uuid dvid.UUID
		if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil {
			return
		}

		switch subcommand {
		case "new":
			var typename, dataname string
			cmd.CommandArgs(3, &typename, &dataname)

			// Get TypeService
			var typeservice datastore.TypeService
			if typeservice, err = datastore.TypeServiceByName(dvid.TypeString(typename)); err != nil {
				return
			}

			// Create new data
			config := cmd.Settings()
			if _, err = datastore.NewData(uuid, typeservice, dvid.InstanceName(dataname), config); err != nil {
				return
			}
			reply.Text = fmt.Sprintf("Data %q [%s] added to node %s\n", dataname, typename, uuid)
			datastore.AddToRepoLog(uuid, []string{cmd.String()})

		case "rename":
			var name1, name2, passcode string
			cmd.CommandArgs(3, &name1, &name2, &passcode)
			oldname := dvid.InstanceName(name1)
			newname := dvid.InstanceName(name2)

			// Make sure this instance exists.
			if _, err = datastore.GetDataByUUIDName(uuid, oldname); err != nil {
				err = fmt.Errorf("Error trying to rename %q for UUID %s: %v", oldname, uuid, err)
				return
			}

			// Do the rename.
			if err = datastore.RenameData(uuid, oldname, newname, passcode); err != nil {
				err = fmt.Errorf("Error renaming data instance %q to %q: %v", oldname, newname, err)
				return
			}
			reply.Text = fmt.Sprintf("Renamed data instance %q to %q from DAG subgraph @ root %s\n", oldname, newname, uuid)

		case "branch":
			cmd.CommandArgs(3, &uuidStr)

			var assign *dvid.UUID
			if uuidStr == "" {
				assign = nil
			} else {
				u := dvid.UUID(uuidStr)
				assign = &u
			}
			var child dvid.UUID
			if child, err = datastore.NewVersion(uuid, fmt.Sprintf("branch of %s", uuid), assign); err != nil {
				return
			}
			reply.Text = fmt.Sprintf("Branch %s added to node %s\n", child, uuid)
			datastore.AddToRepoLog(uuid, []string{cmd.String()})

		case "merge":
			uuids := cmd.CommandArgs(2)

			parents := make([]dvid.UUID, len(uuids)+1)
			parents[0] = dvid.UUID(uuid)
			i := 1
			for uuid := range uuids {
				parents[i] = dvid.UUID(uuid)
				i++
			}
			var child dvid.UUID
			child, err = datastore.Merge(parents, fmt.Sprintf("merge of parents %v", parents), datastore.MergeConflictFree)
			if err != nil {
				return
			}
			reply.Text = fmt.Sprintf("Parents %v merged into node %s\n", parents, child)
			datastore.AddToRepoLog(uuid, []string{cmd.String()})

		case "migrate":
			var source, oldStoreName string
			cmd.CommandArgs(3, &source, &oldStoreName)
			var store dvid.Store
			store, err = storage.GetStoreByAlias(storage.Alias(oldStoreName))
			if err != nil {
				return
			}
			config := cmd.Settings()
			go func() {
				if err = datastore.MigrateInstance(uuid, dvid.InstanceName(source), store, config); err != nil {
					dvid.Errorf("migrate error: %v\n", err)
				}
			}()
			reply.Text = fmt.Sprintf("Started migration of uuid %s data instance %q from old store %q...\n", uuid, source, oldStoreName)

		case "copy":
			var source, target string
			cmd.CommandArgs(3, &source, &target)
			config := cmd.Settings()
			go func() {
				if err = datastore.CopyInstance(uuid, dvid.InstanceName(source), dvid.InstanceName(target), config); err != nil {
					dvid.Errorf("copy error: %v\n", err)
				}
			}()
			reply.Text = fmt.Sprintf("Started copy of uuid %s data instance %q to %q...\n", uuid, source, target)

		case "push":
			var target string
			cmd.CommandArgs(3, &target)
			config := cmd.Settings()
			go func() {
				if err = datastore.PushRepo(uuid, target, config); err != nil {
					dvid.Errorf("push error: %v\n", err)
				}
			}()
			reply.Text = fmt.Sprintf("Started push of repo %s to %q...\n", uuid, target)

			/*
				case "pull":
					var target string
					cmd.CommandArgs(3, &target)
					config := cmd.Settings()
					if err = datastore.Pull(uuid, target, config); err != nil {
						return
					}
					reply.Text = fmt.Sprintf("Repo %s pulled from %q\n", uuid, target)
			*/

		case "delete":
			var dataname, passcode string
			cmd.CommandArgs(3, &dataname, &passcode)

			// Make sure this instance exists.
			if _, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(dataname)); err != nil {
				err = fmt.Errorf("Error trying to delete %q for UUID %s: %v", dataname, uuid, err)
				return
			}

			// Do the deletion.  Under hood, modifies metadata immediately and launches async k/v deletion.
			if err = datastore.DeleteDataByName(uuid, dvid.InstanceName(dataname), passcode); err != nil {
				err = fmt.Errorf("Error deleting data instance %q: %v", dataname, err)
				return
			}
			reply.Text = fmt.Sprintf("Started deletion of data instance %q from repo with root %s\n", dataname, uuid)

		default:
			err = fmt.Errorf("Unknown command: %q", cmd)
			return
		}

	case "node":
		var uuidStr, descriptor string
		cmd.CommandArgs(1, &uuidStr, &descriptor)
		var uuid dvid.UUID
		if uuid, _, err = datastore.MatchingUUID(uuidStr); err != nil {
			return
		}

		// Get the DataService
		dataname := dvid.InstanceName(descriptor)
		var subcommand string
		cmd.CommandArgs(3, &subcommand)
		var dataservice datastore.DataService
		if dataservice, err = datastore.GetDataByUUIDName(uuid, dataname); err != nil {
			return
		}
		if subcommand == "help" {
			reply.Text = dataservice.Help()
			return
		}
		err = dataservice.DoRPC(*cmd, reply)
		return

	default:
		// Check to see if it's a name of a compiled data type, in which case we refer it to the data type.
		types := datastore.CompiledTypes()
		for name, typeservice := range types {
			if name == dvid.TypeString(cmd.Argument(0)) {
				err = typeservice.Do(*cmd, reply)
				return
			}
		}

		err = fmt.Errorf("Unknown command: '%s'", *cmd)
	}
	return
}
Esempio n. 26
0
// CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale
func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error {
	timedLog := dvid.NewTimeLog()

	// Parse the request
	var uuidStr, dataName, cmdStr, grayscaleName, destName string
	request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName)

	// Get the version
	uuid, v, err := datastore.MatchingUUID(uuidStr)
	if err != nil {
		return err
	}

	// Log request
	if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
		return err
	}

	// Get the grayscale data.
	dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(grayscaleName))
	if err != nil {
		return err
	}
	grayscale, ok := dataservice.(*imageblk.Data)
	if !ok {
		return fmt.Errorf("%s is not the name of uint8 data", grayscaleName)
	}

	// Create a new rgba8blk data.
	var compservice datastore.DataService
	compservice, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(destName))
	if err == nil {
		return fmt.Errorf("Data instance with name %q already exists", destName)
	}
	typeService, err := datastore.TypeServiceByName("rgba8blk")
	if err != nil {
		return fmt.Errorf("Could not get rgba8 type service from DVID")
	}
	config := dvid.NewConfig()
	compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config)
	if err != nil {
		return err
	}
	composite, ok := compservice.(*imageblk.Data)
	if !ok {
		return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName)
	}

	// Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time.
	wg := new(sync.WaitGroup)
	op := &compositeOp{grayscale, composite, v}
	chunkOp := &storage.ChunkOp{op, wg}

	store, err := d.GetOrderedKeyValueDB()
	if err != nil {
		return err
	}
	ctx := datastore.NewVersionedCtx(d, v)
	extents := d.Extents()
	blockBeg := imageblk.NewTKey(extents.MinIndex)
	blockEnd := imageblk.NewTKey(extents.MaxIndex)
	err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk))
	wg.Wait()

	// Set new mapped data to same extents.
	composite.Properties.Extents = grayscale.Properties.Extents
	if err := datastore.SaveDataByUUID(uuid, composite); err != nil {
		dvid.Infof("Could not save new data '%s': %v\n", destName, err)
	}

	timedLog.Infof("Created composite of %s and %s", grayscaleName, destName)
	return nil
}