func (vol *labelVol) testBlocks(t *testing.T, uuid dvid.UUID, compression, roi string) {
span := 5
apiStr := fmt.Sprintf("%snode/%s/%s/blocks/%d_%d_%d/%d_%d_%d", server.WebAPIPath,
uuid, vol.name, 160, 32, 32, vol.offset[0], vol.offset[1], vol.offset[2])
if compression == "uncompressed" {
apiStr += "?compression=uncompressed"
}
data := server.TestHTTP(t, "GET", apiStr, nil)
fmt.Printf("Got %d bytes of data\n", len(data))
blockBytes := 32 * 32 * 32 * 8
// Check if values are what we expect
bx := vol.offset[0] / 32
by := vol.offset[1] / 32
bz := vol.offset[2] / 32
b := 0
for i := 0; i < span; i++ {
// Get block coord + block size
if b+16 > len(data) {
t.Fatalf("Only got %d bytes back from block API call, yet next coord in span would put index @ %d\n", len(data), b+16)
}
x := int32(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
y := int32(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
z := int32(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
n := int(binary.LittleEndian.Uint32(data[b : b+4]))
b += 4
if x != bx || y != by || z != bz {
t.Errorf("Bad block coordinate: expected (%d,%d,%d), got (%d,%d,%d)\n", bx, by, bz, x, y, z)
}
// Read in the block data as assumed LZ4 and check it.
var uncompressed []byte
if compression != "uncompressed" {
uncompressed = make([]byte, blockBytes)
if err := lz4.Uncompress(data[b:b+n], uncompressed); err != nil {
t.Fatalf("Unable to uncompress LZ4 data (%s), %d bytes: %v\n", apiStr, n, err)
}
} else {
uncompressed = data[b : b+n]
}
vol.testBlock(t, x, y, z, uncompressed)
b += n
bx++
}
}
func (vol labelVol) getLabelVolume(t *testing.T, uuid dvid.UUID, compression, roi string) []byte {
apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", server.WebAPIPath,
uuid, vol.name, vol.nx, vol.ny, vol.nz, vol.offset[0], vol.offset[1], vol.offset[2])
query := true
switch compression {
case "lz4":
apiStr += "?compression=lz4"
case "gzip":
apiStr += "?compression=gzip"
default:
query = false
}
if roi != "" {
if query {
apiStr += "&roi=" + roi
} else {
apiStr += "?roi=" + roi
}
}
data := server.TestHTTP(t, "GET", apiStr, nil)
switch compression {
case "lz4":
uncompressed := make([]byte, vol.numBytes())
if err := lz4.Uncompress(data, uncompressed); err != nil {
t.Fatalf("Unable to uncompress LZ4 data (%s), %d bytes: %v\n", apiStr, len(data), err)
}
data = uncompressed
case "gzip":
buf := bytes.NewBuffer(data)
gr, err := gzip.NewReader(buf)
if err != nil {
t.Fatalf("Error on gzip new reader: %v\n", err)
}
uncompressed, err := ioutil.ReadAll(gr)
if err != nil {
t.Fatalf("Error on reading gzip: %v\n", err)
}
if err = gr.Close(); err != nil {
t.Fatalf("Error on closing gzip: %v\n", err)
}
data = uncompressed
default:
}
if len(data) != int(vol.numBytes()) {
t.Errorf("Expected %d uncompressed bytes from 3d labelblk GET. Got %d instead.", vol.numBytes(), len(data))
}
return data
}
// DeserializeData deserializes a slice of bytes using stored compression, checksum.
// If uncompress parameter is false, the data is not uncompressed.
func DeserializeData(s []byte, uncompress bool) ([]byte, CompressionFormat, error) {
if s == nil || len(s) == 0 {
return []byte{}, Uncompressed, nil
}
buffer := bytes.NewBuffer(s)
// Get the stored compression and checksum
var format SerializationFormat
if err := binary.Read(buffer, binary.LittleEndian, &format); err != nil {
return nil, 0, fmt.Errorf("Could not read serialization format info from %d byte input: %v", len(s), err)
}
compression, checksum := DecodeSerializationFormat(format)
// Get any checksum.
var storedCrc32 uint32
switch checksum {
case NoChecksum:
case CRC32:
if err := binary.Read(buffer, binary.LittleEndian, &storedCrc32); err != nil {
return nil, 0, fmt.Errorf("Error reading checksum: %v", err)
}
default:
return nil, 0, fmt.Errorf("Illegal checksum in deserializing data")
}
// Get the possibly compressed data.
cdata := buffer.Bytes()
// Perform any requested checksum
switch checksum {
case CRC32:
crcChecksum := crc32.ChecksumIEEE(cdata)
if crcChecksum != storedCrc32 {
return nil, 0, fmt.Errorf("Bad checksum. Stored %x got %x", storedCrc32, crcChecksum)
}
}
// Return data with optional compression
if !uncompress || compression == Uncompressed {
return cdata, compression, nil
}
switch compression {
case Snappy:
data, err := snappy.Decode(nil, cdata)
if err != nil {
return nil, 0, err
}
return data, compression, nil
case LZ4:
origSize := binary.LittleEndian.Uint32(cdata[0:4])
data := make([]byte, int(origSize))
if err := lz4.Uncompress(cdata[4:], data); err != nil {
return nil, 0, err
}
return data, compression, nil
case Gzip:
b := bytes.NewBuffer(cdata)
var err error
r, err := gzip.NewReader(b)
if err != nil {
return nil, 0, err
}
var buffer bytes.Buffer
_, err = io.Copy(&buffer, r)
if err != nil {
return nil, 0, err
}
err = r.Close()
if err != nil {
return nil, 0, err
}
return buffer.Bytes(), compression, nil
default:
return nil, 0, fmt.Errorf("Illegal compression format (%d) in deserialization", compression)
}
}
// ServeHTTP handles all incoming HTTP requests for this data.
func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) {
// TODO -- Refactor this method to break it up and make it simpler. Use the web routing for the endpoints.
timedLog := dvid.NewTimeLog()
// Get the action (GET, POST)
action := strings.ToLower(r.Method)
switch action {
case "get":
case "post":
case "delete":
default:
server.BadRequest(w, r, "labelblk only handles GET, POST, and DELETE HTTP verbs")
return
}
// Break URL request into arguments
url := r.URL.Path[len(server.WebAPIPath):]
parts := strings.Split(url, "/")
if len(parts[len(parts)-1]) == 0 {
parts = parts[:len(parts)-1]
}
// Get query strings and possible roi
var roiptr *imageblk.ROI
queryValues := r.URL.Query()
roiname := dvid.InstanceName(queryValues.Get("roi"))
if len(roiname) != 0 {
roiptr = new(imageblk.ROI)
}
// Handle POST on data -> setting of configuration
if len(parts) == 3 && action == "post" {
config, err := server.DecodeJSON(r)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err := d.ModifyConfig(config); err != nil {
server.BadRequest(w, r, err)
return
}
if err := datastore.SaveDataByUUID(uuid, d); err != nil {
server.BadRequest(w, r, err)
return
}
fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
return
}
if len(parts) < 4 {
server.BadRequest(w, r, "Incomplete API request")
return
}
// Process help and info.
switch parts[3] {
case "help":
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, dtype.Help())
case "metadata":
jsonStr, err := d.NdDataMetadata()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json")
fmt.Fprintln(w, jsonStr)
case "info":
jsonBytes, err := d.MarshalJSON()
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, string(jsonBytes))
case "label":
// GET <api URL>/node/<UUID>/<data name>/label/<coord>
if len(parts) < 5 {
server.BadRequest(w, r, "DVID requires coord to follow 'label' command")
return
}
coord, err := dvid.StringToPoint(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/json")
jsonStr := fmt.Sprintf(`{"Label": %d}`, label)
fmt.Fprintf(w, jsonStr)
timedLog.Infof("HTTP %s: label at %s (%s)", r.Method, coord, r.URL)
case "labels":
// POST <api URL>/node/<UUID>/<data name>/labels
if action != "post" {
server.BadRequest(w, r, "Batch labels query must be a POST request")
return
}
data, err := ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, "Bad POSTed data for batch reverse query. Should be JSON.")
return
}
var coords []dvid.Point3d
if err := json.Unmarshal(data, &coords); err != nil {
server.BadRequest(w, r, fmt.Sprintf("Bad labels request JSON: %v", err))
return
}
w.Header().Set("Content-type", "application/json")
fmt.Fprintf(w, "[")
sep := false
for _, coord := range coords {
label, err := d.GetLabelAtPoint(ctx.VersionID(), coord)
if err != nil {
server.BadRequest(w, r, err)
return
}
if sep {
fmt.Fprintf(w, ",")
}
fmt.Fprintf(w, "%d", label)
sep = true
}
fmt.Fprintf(w, "]")
timedLog.Infof("HTTP batch label-at-point query (%d coordinates)", len(coords))
case "blocks":
// DELETE <api URL>/node/<UUID>/<data name>/blocks/<block coord>/<spanX>
if len(parts) < 6 {
server.BadRequest(w, r, "DVID requires starting block coord and # of blocks along X to follow 'blocks' endpoint.")
return
}
blockCoord, err := dvid.StringToChunkPoint3d(parts[4], "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
span, err := strconv.Atoi(parts[5])
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "delete" {
server.BadRequest(w, r, "DVID currently accepts only DELETE on the 'blocks' endpoint")
return
}
if err := d.DeleteBlocks(ctx, blockCoord, span); err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: delete %d blocks from %s (%s)", r.Method, span, blockCoord, r.URL)
case "pseudocolor":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
lbl, err := d.NewLabels(slice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
// Convert to pseudocolor
pseudoColor, err := colorImage(img)
if err != nil {
server.BadRequest(w, r, err)
return
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
err = dvid.WriteImageHttp(w, pseudoColor, formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports only 2d pseudocolor image requests")
return
}
case "raw", "isotropic":
if len(parts) < 7 {
server.BadRequest(w, r, "'%s' must be followed by shape/size/offset", parts[3])
return
}
var isotropic bool = (parts[3] == "isotropic")
shapeStr, sizeStr, offsetStr := parts[4], parts[5], parts[6]
planeStr := dvid.DataShapeString(shapeStr)
plane, err := planeStr.DataShape()
if err != nil {
server.BadRequest(w, r, err)
return
}
switch plane.ShapeDimensions() {
case 2:
slice, err := dvid.NewSliceFromStrings(planeStr, offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action != "get" {
server.BadRequest(w, r, "DVID does not permit 2d mutations, only 3d block-aligned stores")
return
}
rawSlice, err := dvid.Isotropy2D(d.Properties.VoxelSize, slice, isotropic)
lbl, err := d.NewLabels(rawSlice, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
img, err := d.GetImage(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if isotropic {
dstW := int(slice.Size().Value(0))
dstH := int(slice.Size().Value(1))
img, err = img.ScaleImage(dstW, dstH)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
var formatStr string
if len(parts) >= 8 {
formatStr = parts[7]
}
//dvid.ElapsedTime(dvid.Normal, startTime, "%s %s upto image formatting", op, slice)
err = dvid.WriteImageHttp(w, img.Get(), formatStr)
if err != nil {
server.BadRequest(w, r, err)
return
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, plane, r.URL)
case 3:
queryStrings := r.URL.Query()
throttle := queryStrings.Get("throttle")
if throttle == "true" || throttle == "on" {
select {
case <-server.Throttle:
// Proceed with operation, returning throttle token to server at end.
defer func() {
server.Throttle <- 1
}()
default:
throttleMsg := fmt.Sprintf("Server already running maximum of %d throttled operations",
server.MaxThrottledOps)
http.Error(w, throttleMsg, http.StatusServiceUnavailable)
return
}
}
compression := queryValues.Get("compression")
subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
if err != nil {
server.BadRequest(w, r, err)
return
}
if action == "get" {
lbl, err := d.NewLabels(subvol, nil)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
data, err := d.GetVolume(ctx.VersionID(), lbl, roiptr)
if err != nil {
server.BadRequest(w, r, err)
return
}
w.Header().Set("Content-type", "application/octet-stream")
switch compression {
case "":
_, err = w.Write(data)
if err != nil {
server.BadRequest(w, r, err)
return
}
case "lz4":
compressed := make([]byte, lz4.CompressBound(data))
var n, outSize int
if outSize, err = lz4.Compress(data, compressed); err != nil {
server.BadRequest(w, r, err)
return
}
compressed = compressed[:outSize]
if n, err = w.Write(compressed); err != nil {
server.BadRequest(w, r, err)
return
}
if n != outSize {
errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
dvid.Errorf(errmsg)
server.BadRequest(w, r, errmsg)
return
}
case "gzip":
gw := gzip.NewWriter(w)
if _, err = gw.Write(data); err != nil {
server.BadRequest(w, r, err)
return
}
if err = gw.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
} else {
if isotropic {
server.BadRequest(w, r, "can only POST 'raw' not 'isotropic' images")
return
}
// Make sure vox is block-aligned
if !dvid.BlockAligned(subvol, d.BlockSize()) {
server.BadRequest(w, r, "cannot store labels in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
return
}
var data []byte
switch compression {
case "":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read 3d uncompressed POST")
case "lz4":
tlog := dvid.NewTimeLog()
data, err = ioutil.ReadAll(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
if len(data) == 0 {
server.BadRequest(w, r, "received 0 LZ4 compressed bytes")
return
}
tlog.Debugf("read 3d lz4 POST")
tlog = dvid.NewTimeLog()
uncompressed := make([]byte, subvol.NumVoxels()*8)
err = lz4.Uncompress(data, uncompressed)
if err != nil {
server.BadRequest(w, r, err)
return
}
data = uncompressed
tlog.Debugf("uncompressed 3d lz4 POST")
case "gzip":
tlog := dvid.NewTimeLog()
gr, err := gzip.NewReader(r.Body)
if err != nil {
server.BadRequest(w, r, err)
return
}
data, err = ioutil.ReadAll(gr)
if err != nil {
server.BadRequest(w, r, err)
return
}
if err = gr.Close(); err != nil {
server.BadRequest(w, r, err)
return
}
tlog.Debugf("read and uncompress 3d gzip POST")
default:
server.BadRequest(w, r, "unknown compression type %q", compression)
return
}
lbl, err := d.NewLabels(subvol, data)
if err != nil {
server.BadRequest(w, r, err)
return
}
if roiptr != nil {
roiptr.Iter, err = roi.NewIterator(roiname, ctx.VersionID(), lbl)
if err != nil {
server.BadRequest(w, r, err)
return
}
}
if err = d.PutVoxels(ctx.VersionID(), lbl.Voxels, roiptr); err != nil {
server.BadRequest(w, r, err)
return
}
}
timedLog.Infof("HTTP %s: %s (%s)", r.Method, subvol, r.URL)
default:
server.BadRequest(w, r, "DVID currently supports shapes of only 2 and 3 dimensions")
return
}
default:
server.BadRequest(w, r, "Unrecognized API call %q for labelblk data %q. See API help.",
parts[3], d.DataName())
}
}