示例#1
0
// DownRes downsamples 2d Voxels data by averaging where the down-magnification are
// integers.  If the source image size in Voxels is not an integral multiple of the
// reduction factor, the edge voxels on the right and bottom side are truncated.
// This function modifies the Voxels data.  An error is returned if a non-2d Voxels
// receiver is used.
func (v *Voxels) DownRes(magnification dvid.Point) error {
	if v.DataShape().ShapeDimensions() != 2 {
		return fmt.Errorf("ImageDownres() only supports 2d images at this time.")
	}
	// Calculate new dimensions and allocate.
	srcW := v.Size().Value(0)
	srcH := v.Size().Value(1)
	reduceW, reduceH, err := v.DataShape().GetSize2D(magnification)
	if err != nil {
		return err
	}
	dstW := srcW / reduceW
	dstH := srcH / reduceH

	// Reduce the image.
	img, err := v.GetImage2d()
	if err != nil {
		return err
	}
	img, err = img.ScaleImage(int(dstW), int(dstH))
	if err != nil {
		return err
	}

	// Set data and dimensions to downres data.
	v.data = []byte(img.Data())
	geom, err := dvid.NewOrthogSlice(v.DataShape(), v.StartPoint(), dvid.Point2d{dstW, dstH})
	if err != nil {
		return err
	}
	v.Geometry = geom
	v.stride = dstW * v.values.BytesPerElement()
	return nil
}
示例#2
0
文件: ingest.go 项目: hanslovsky/dvid
// Loads a XY oriented image at given offset, returning Voxels.
func (d *Data) loadXYImage(filename string, offset dvid.Point) (*Voxels, error) {
	img, _, err := dvid.GoImageFromFile(filename)
	if err != nil {
		return nil, err
	}
	slice, err := dvid.NewOrthogSlice(dvid.XY, offset, dvid.RectSize(img.Bounds()))
	if err != nil {
		return nil, fmt.Errorf("Unable to determine slice: %v", err)
	}
	vox, err := d.NewVoxels(slice, img)
	if err != nil {
		return nil, err
	}
	storage.FileBytesRead <- len(vox.Data())
	return vox, nil
}
示例#3
0
func (d *Data) ConstructTiles(uuidStr string, tileSpec TileSpec, request datastore.Request) error {
	config := request.Settings()
	uuid, versionID, err := datastore.MatchingUUID(uuidStr)
	if err != nil {
		return err
	}
	if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
		return err
	}

	source, err := datastore.GetDataByUUID(uuid, d.Source)
	if err != nil {
		return err
	}
	src, ok := source.(*imageblk.Data)
	if !ok {
		return fmt.Errorf("Cannot construct imagetile for non-voxels data: %s", d.Source)
	}

	// Save the current tile specification
	d.Levels = tileSpec
	if err := datastore.SaveDataByUUID(uuid, d); err != nil {
		return err
	}

	// Get size of tile at lowest resolution.
	lastLevel := Scaling(len(tileSpec) - 1)
	loresSpec, found := tileSpec[lastLevel]
	if !found {
		return fmt.Errorf("Illegal tile spec.  Should have levels 0 to absent %d.", lastLevel)
	}
	var loresSize [3]float64
	for i := 0; i < 3; i++ {
		loresSize[i] = float64(loresSpec.Resolution[i]) * float64(DefaultTileSize[i])
	}
	loresMag := dvid.Point3d{1, 1, 1}
	for i := Scaling(0); i < lastLevel; i++ {
		levelMag := tileSpec[i].levelMag
		loresMag[0] *= levelMag[0]
		loresMag[1] *= levelMag[1]
		loresMag[2] *= levelMag[2]
	}

	// Get min and max points in terms of distance.
	var minPtDist, maxPtDist [3]float64
	for i := uint8(0); i < 3; i++ {
		minPtDist[i] = float64(src.MinPoint.Value(i)) * float64(src.VoxelSize[i])
		maxPtDist[i] = float64(src.MaxPoint.Value(i)) * float64(src.VoxelSize[i])
	}

	// Adjust min and max points for the tileable surface at lowest resolution.
	var minTiledPt, maxTiledPt dvid.Point3d
	for i := 0; i < 3; i++ {
		minInt, _ := math.Modf(minPtDist[i] / loresSize[i])
		maxInt, _ := math.Modf(maxPtDist[i] / loresSize[i])
		minTileCoord := int32(minInt)
		maxTileCoord := int32(maxInt)
		minTiledPt[i] = minTileCoord * DefaultTileSize[i] * loresMag[i]
		maxTiledPt[i] = (maxTileCoord+1)*DefaultTileSize[i]*loresMag[i] - 1
	}
	sizeVolume := maxTiledPt.Sub(minTiledPt).AddScalar(1)

	// Setup swappable ExtData buffers (the stitched slices) so we can be generating tiles
	// at same time we are reading and stitching them.
	var bufferLock [2]sync.Mutex
	var sliceBuffers [2]*imageblk.Voxels
	var bufferNum int

	// Get the planes we should tile.
	planes, err := config.GetShapes("planes", ";")
	if planes == nil {
		// If no planes are specified, construct imagetile for 3 orthogonal planes.
		planes = []dvid.DataShape{dvid.XY, dvid.XZ, dvid.YZ}
	}

	outF, err := d.putTileFunc(versionID)

	// sort the tile spec keys to iterate from highest to lowest resolution
	var sortedKeys []int
	for scaling, _ := range tileSpec {
		sortedKeys = append(sortedKeys, int(scaling))
	}
	sort.Ints(sortedKeys)

	for _, plane := range planes {
		timedLog := dvid.NewTimeLog()
		offset := minTiledPt.Duplicate()

		switch {

		case plane.Equals(dvid.XY):
			width, height, err := plane.GetSize2D(sizeVolume)
			if err != nil {
				return err
			}
			dvid.Debugf("Tiling XY image %d x %d pixels\n", width, height)
			for z := src.MinPoint.Value(2); z <= src.MaxPoint.Value(2); z++ {
				server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xy]")

				sliceLog := dvid.NewTimeLog()
				offset = offset.Modify(map[uint8]int32{2: z})
				slice, err := dvid.NewOrthogSlice(dvid.XY, offset, dvid.Point2d{width, height})
				if err != nil {
					return err
				}
				bufferLock[bufferNum].Lock()
				sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil)
				if err != nil {
					return err
				}
				if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil {
					return err
				}
				// Iterate through the different scales, extracting tiles at each resolution.
				go func(bufferNum int, offset dvid.Point) {
					defer bufferLock[bufferNum].Unlock()
					timedLog := dvid.NewTimeLog()
					for _, key := range sortedKeys {
						scaling := Scaling(key)
						levelSpec := tileSpec[scaling]
						if err != nil {
							dvid.Errorf("Error in tiling: %v\n", err)
							return
						}
						if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil {
							dvid.Errorf("Error in tiling: %v\n", err)
							return
						}
						if int(scaling) < len(tileSpec)-1 {
							if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil {
								dvid.Errorf("Error in tiling: %v\n", err)
								return
							}
						}
					}
					timedLog.Debugf("Tiled XY Tile using buffer %d", bufferNum)
				}(bufferNum, offset)

				sliceLog.Infof("Read XY Tile @ Z = %d, now tiling...", z)
				bufferNum = (bufferNum + 1) % 2
			}
			timedLog.Infof("Total time to generate XY Tiles")

		case plane.Equals(dvid.XZ):
			width, height, err := plane.GetSize2D(sizeVolume)
			if err != nil {
				return err
			}
			dvid.Debugf("Tiling XZ image %d x %d pixels\n", width, height)
			for y := src.MinPoint.Value(1); y <= src.MaxPoint.Value(1); y++ {
				server.BlockOnInteractiveRequests("imagetile.ConstructTiles [xz]")

				sliceLog := dvid.NewTimeLog()
				offset = offset.Modify(map[uint8]int32{1: y})
				slice, err := dvid.NewOrthogSlice(dvid.XZ, offset, dvid.Point2d{width, height})
				if err != nil {
					return err
				}
				bufferLock[bufferNum].Lock()
				sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil)
				if err != nil {
					return err
				}
				if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil {
					return err
				}
				// Iterate through the different scales, extracting tiles at each resolution.
				go func(bufferNum int, offset dvid.Point) {
					defer bufferLock[bufferNum].Unlock()
					timedLog := dvid.NewTimeLog()
					for _, key := range sortedKeys {
						scaling := Scaling(key)
						levelSpec := tileSpec[scaling]
						if err != nil {
							dvid.Errorf("Error in tiling: %v\n", err)
							return
						}
						if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil {
							dvid.Errorf("Error in tiling: %v\n", err)
							return
						}
						if int(scaling) < len(tileSpec)-1 {
							if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil {
								dvid.Errorf("Error in tiling: %v\n", err)
								return
							}
						}
					}
					timedLog.Debugf("Tiled XZ Tile using buffer %d", bufferNum)
				}(bufferNum, offset)

				sliceLog.Infof("Read XZ Tile @ Y = %d, now tiling...", y)
				bufferNum = (bufferNum + 1) % 2
			}
			timedLog.Infof("Total time to generate XZ Tiles")

		case plane.Equals(dvid.YZ):
			width, height, err := plane.GetSize2D(sizeVolume)
			if err != nil {
				return err
			}
			dvid.Debugf("Tiling YZ image %d x %d pixels\n", width, height)
			for x := src.MinPoint.Value(0); x <= src.MaxPoint.Value(0); x++ {
				server.BlockOnInteractiveRequests("imagetile.ConstructTiles [yz]")

				sliceLog := dvid.NewTimeLog()
				offset = offset.Modify(map[uint8]int32{0: x})
				slice, err := dvid.NewOrthogSlice(dvid.YZ, offset, dvid.Point2d{width, height})
				if err != nil {
					return err
				}
				bufferLock[bufferNum].Lock()
				sliceBuffers[bufferNum], err = src.NewVoxels(slice, nil)
				if err != nil {
					return err
				}
				if err = src.GetVoxels(versionID, sliceBuffers[bufferNum], nil); err != nil {
					return err
				}
				// Iterate through the different scales, extracting tiles at each resolution.
				go func(bufferNum int, offset dvid.Point) {
					defer bufferLock[bufferNum].Unlock()
					timedLog := dvid.NewTimeLog()
					for _, key := range sortedKeys {
						scaling := Scaling(key)
						levelSpec := tileSpec[scaling]
						outF, err := d.putTileFunc(versionID)
						if err != nil {
							dvid.Errorf("Error in tiling: %v\n", err)
							return
						}
						if err := d.extractTiles(sliceBuffers[bufferNum], offset, scaling, outF); err != nil {
							dvid.Errorf("Error in tiling: %v\n", err)
							return
						}
						if int(scaling) < len(tileSpec)-1 {
							if err := sliceBuffers[bufferNum].DownRes(levelSpec.levelMag); err != nil {
								dvid.Errorf("Error in tiling: %v\n", err)
								return
							}
						}
					}
					timedLog.Debugf("Tiled YZ Tile using buffer %d", bufferNum)
				}(bufferNum, offset)

				sliceLog.Debugf("Read YZ Tile @ X = %d, now tiling...", x)
				bufferNum = (bufferNum + 1) % 2
			}
			timedLog.Infof("Total time to generate YZ Tiles")

		default:
			dvid.Infof("Skipping request to tile '%s'.  Unsupported.", plane)
		}
	}
	return nil
}
示例#4
0
// PutLocal adds image data to a version node, altering underlying blocks if the image
// intersects the block.
//
// The image filename glob MUST BE absolute file paths that are visible to the server.
// This function is meant for mass ingestion of large data files, and it is inappropriate
// to read gigabytes of data just to send it over the network to a local DVID.
func (d *Data) PutLocal(request datastore.Request, reply *datastore.Response) error {
	timedLog := dvid.NewTimeLog()

	// Parse the request
	var uuidStr, dataName, cmdStr, sourceStr, planeStr, offsetStr string
	filenames := request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &sourceStr,
		&planeStr, &offsetStr)
	if len(filenames) == 0 {
		return fmt.Errorf("Need to include at least one file to add: %s", request)
	}

	// Get offset
	offset, err := dvid.StringToPoint(offsetStr, ",")
	if err != nil {
		return fmt.Errorf("Illegal offset specification: %s: %v", offsetStr, err)
	}

	// Get list of files to add
	var addedFiles string
	if len(filenames) == 1 {
		addedFiles = filenames[0]
	} else {
		addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1)
	}
	dvid.Debugf(addedFiles + "\n")

	// Get plane
	plane, err := dvid.DataShapeString(planeStr).DataShape()
	if err != nil {
		return err
	}

	// Get Repo and IDs
	uuid, versionID, err := datastore.MatchingUUID(uuidStr)
	if err != nil {
		return err
	}

	// Load and PUT each image.
	numSuccessful := 0
	for _, filename := range filenames {
		sliceLog := dvid.NewTimeLog()
		img, _, err := dvid.GoImageFromFile(filename)
		if err != nil {
			return fmt.Errorf("Error after %d images successfully added: %v", numSuccessful, err)
		}
		slice, err := dvid.NewOrthogSlice(plane, offset, dvid.RectSize(img.Bounds()))
		if err != nil {
			return fmt.Errorf("Unable to determine slice: %v", err)
		}

		vox, err := d.NewVoxels(slice, img)
		if err != nil {
			return err
		}
		storage.FileBytesRead <- len(vox.Data())
		if err = d.PutVoxels(versionID, vox, nil); err != nil {
			return err
		}
		sliceLog.Debugf("%s put local %s", d.DataName(), slice)
		numSuccessful++
		offset = offset.Add(dvid.Point3d{0, 0, 1})
	}

	if err := datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
		return err
	}
	timedLog.Infof("RPC put local (%s) completed", addedFiles)
	return nil
}