// Deletes an ROI. func (d *Data) Delete(ctx storage.VersionedCtx) error { smalldata, err := storage.SmallDataStore() if err != nil { return err } // We only want one PUT on given version for given data to prevent interleaved PUTs. putMutex := ctx.Mutex() putMutex.Lock() defer func() { putMutex.Unlock() }() d.MinZ = math.MaxInt32 d.MaxZ = math.MinInt32 if err := datastore.SaveDataByVersion(ctx.VersionID(), d); err != nil { return fmt.Errorf("Error in trying to save repo on roi extent change: %v\n", err) } // Delete all spans for this ROI for just this version. if err := smalldata.DeleteAll(ctx, false); err != nil { return err } return nil }
// LoadImages bulk loads images using different techniques if it is a multidimensional // file like HDF5 or a sequence of PNG/JPG/TIF images. func (d *Data) LoadImages(v dvid.VersionID, offset dvid.Point, filenames []string) error { if len(filenames) == 0 { return nil } timedLog := dvid.NewTimeLog() // We only want one PUT on given version for given data to prevent interleaved // chunk PUTs that could potentially overwrite slice modifications. ctx := storage.NewDataContext(d, v) loadMutex := ctx.Mutex() loadMutex.Lock() // Handle cleanup given multiple goroutines still writing data. load := &bulkLoadInfo{filenames: filenames, versionID: v, offset: offset} defer func() { loadMutex.Unlock() if load.extentChanged.Value() { err := datastore.SaveDataByVersion(v, d) if err != nil { dvid.Errorf("Error in trying to save repo for voxel extent change: %v\n", err) } } }() // Use different loading techniques if we have a potentially multidimensional HDF5 file // or many 2d images. var err error if dvid.Filename(filenames[0]).HasExtensionPrefix("hdf", "h5") { err = d.loadHDF(load) } else { err = d.loadXYImages(load) } if err != nil { timedLog.Infof("RPC load of %d files had error: %v\n", err) } else { timedLog.Infof("RPC load of %d files completed.\n", len(filenames)) } return err }
// PutVoxels persists voxels from a subvolume into the storage engine. // The subvolume must be aligned to blocks of the data instance. // // This requirement simplifies the coding quite a bit. Earlier versions // of DVID allowed 2d writes, which required reading blocks, writing subsets of data // into those blocks, and then writing the result all within a transaction. It is // difficult to scale without requiring GETs within transactions and more complicated // coordination when moving to distributed front-end DVIDs. func (d *Data) PutVoxels(v dvid.VersionID, vox *Voxels, roi *ROI) error { // Make sure vox is block-aligned if !dvid.BlockAligned(vox, d.BlockSize()) { return fmt.Errorf("cannot store voxels in non-block aligned geometry %s -> %s", vox.StartPoint(), vox.EndPoint()) } wg := new(sync.WaitGroup) // Only do one request at a time, although each request can start many goroutines. server.SpawnGoroutineMutex.Lock() defer server.SpawnGoroutineMutex.Unlock() // Keep track of changing extents and mark repo as dirty if changed. var extentChanged bool defer func() { if extentChanged { err := datastore.SaveDataByVersion(v, d) if err != nil { dvid.Infof("Error in trying to save repo on change: %v\n", err) } } }() // Track point extents extents := d.Extents() if extents.AdjustPoints(vox.StartPoint(), vox.EndPoint()) { extentChanged = true } // Iterate through index space for this data. for it, err := vox.IndexIterator(d.BlockSize()); err == nil && it.Valid(); it.NextSpan() { i0, i1, err := it.IndexSpan() if err != nil { return err } ptBeg := i0.Duplicate().(dvid.ChunkIndexer) ptEnd := i1.Duplicate().(dvid.ChunkIndexer) begX := ptBeg.Value(0) endX := ptEnd.Value(0) if extents.AdjustIndices(ptBeg, ptEnd) { extentChanged = true } wg.Add(int(endX-begX) + 1) c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)} for x := begX; x <= endX; x++ { c[0] = x curIndex := dvid.IndexZYX(c) // Don't PUT if this index is outside a specified ROI if roi != nil && roi.Iter != nil && !roi.Iter.InsideFast(curIndex) { wg.Done() continue } kv := &storage.TKeyValue{K: NewTKey(&curIndex)} putOp := &putOperation{vox, curIndex, v} op := &storage.ChunkOp{putOp, wg} d.PutChunk(&storage.Chunk{op, kv}) } } wg.Wait() return nil }
// PutSpans saves a slice of spans representing an ROI into the datastore. // If the init parameter is true, all previous spans of this ROI are deleted before // writing these spans. func (d *Data) PutSpans(versionID dvid.VersionID, spans []dvid.Span, init bool) error { ctx := datastore.NewVersionedCtx(d, versionID) db, err := storage.SmallDataStore() if err != nil { return err } // Delete the old key/values if init { if err := d.Delete(ctx); err != nil { return err } } // Make sure our small data store can do batching. batcher, ok := db.(storage.KeyValueBatcher) if !ok { return fmt.Errorf("Unable to store ROI: small data store can't do batching!") } // We only want one PUT on given version for given data to prevent interleaved PUTs. putMutex := ctx.Mutex() putMutex.Lock() // Save new extents after finished. defer func() { err := datastore.SaveDataByVersion(ctx.VersionID(), d) if err != nil { dvid.Errorf("Error in trying to save repo on roi extent change: %v\n", err) } putMutex.Unlock() }() // Put the new key/values const BATCH_SIZE = 10000 batch := batcher.NewBatch(ctx) for i, span := range spans { if span[0] < d.MinZ { d.MinZ = span[0] } if span[0] > d.MaxZ { d.MaxZ = span[0] } if span[3] < span[2] { return fmt.Errorf("Got weird span %v. span[3] (X1) < span[2] (X0)", span) } index := indexRLE{ start: dvid.IndexZYX{span[2], span[1], span[0]}, span: uint32(span[3] - span[2] + 1), } tk := storage.NewTKey(keyROI, index.Bytes()) batch.Put(tk, dvid.EmptyValue()) if (i+1)%BATCH_SIZE == 0 { if err := batch.Commit(); err != nil { return fmt.Errorf("Error on batch PUT at span %d: %v\n", i, err) } batch = batcher.NewBatch(ctx) } } if len(spans)%BATCH_SIZE != 0 { if err := batch.Commit(); err != nil { return fmt.Errorf("Error on last batch PUT: %v\n", err) } } return nil }
// PutVoxels persists voxels from a subvolume into the storage engine. // The subvolume must be aligned to blocks of the data instance, which simplifies // the routine if the PUT is a mutation (signals MutateBlockEvent) instead of ingestion. func (d *Data) PutVoxels(v dvid.VersionID, mutID uint64, vox *Voxels, roiname dvid.InstanceName, mutate bool) error { r, err := GetROI(v, roiname, vox) if err != nil { return err } // Make sure vox is block-aligned if !dvid.BlockAligned(vox, d.BlockSize()) { return fmt.Errorf("cannot store voxels in non-block aligned geometry %s -> %s", vox.StartPoint(), vox.EndPoint()) } wg := new(sync.WaitGroup) // Only do one request at a time, although each request can start many goroutines. server.SpawnGoroutineMutex.Lock() defer server.SpawnGoroutineMutex.Unlock() // Keep track of changing extents and mark repo as dirty if changed. var extentChanged bool defer func() { if extentChanged { err := datastore.SaveDataByVersion(v, d) if err != nil { dvid.Infof("Error in trying to save repo on change: %v\n", err) } } }() // Track point extents extents := d.Extents() if extents.AdjustPoints(vox.StartPoint(), vox.EndPoint()) { extentChanged = true } // extract buffer interface if it exists var putbuffer storage.RequestBuffer store, err := d.GetOrderedKeyValueDB() if err != nil { return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err) } if req, ok := store.(storage.KeyValueRequester); ok { ctx := datastore.NewVersionedCtx(d, v) putbuffer = req.NewBuffer(ctx) } // Iterate through index space for this data. for it, err := vox.IndexIterator(d.BlockSize()); err == nil && it.Valid(); it.NextSpan() { i0, i1, err := it.IndexSpan() if err != nil { return err } ptBeg := i0.Duplicate().(dvid.ChunkIndexer) ptEnd := i1.Duplicate().(dvid.ChunkIndexer) begX := ptBeg.Value(0) endX := ptEnd.Value(0) if extents.AdjustIndices(ptBeg, ptEnd) { extentChanged = true } wg.Add(int(endX-begX) + 1) c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)} for x := begX; x <= endX; x++ { c[0] = x curIndex := dvid.IndexZYX(c) // Don't PUT if this index is outside a specified ROI if r != nil && r.Iter != nil && !r.Iter.InsideFast(curIndex) { wg.Done() continue } kv := &storage.TKeyValue{K: NewTKey(&curIndex)} putOp := &putOperation{vox, curIndex, v, mutate, mutID} op := &storage.ChunkOp{putOp, wg} d.PutChunk(&storage.Chunk{op, kv}, putbuffer) } } wg.Wait() // if a bufferable op, flush if putbuffer != nil { putbuffer.Flush() } return nil }