// NewLabel returns a new label for the given version.
func (d *Data) NewLabel(v dvid.VersionID) (uint64, error) {
d.ml_mu.Lock()
defer d.ml_mu.Unlock()
// Make sure we aren't trying to increment a label on a locked node.
locked, err := datastore.LockedVersion(v)
if err != nil {
return 0, err
}
if locked {
return 0, fmt.Errorf("can't ask for new label in a locked version id %d", v)
}
// Increment and store.
d.MaxRepoLabel++
d.MaxLabel[v] = d.MaxRepoLabel
store, err := storage.SmallDataStore()
if err != nil {
return 0, fmt.Errorf("can't initializing small data store: %v\n", err)
}
buf := make([]byte, 8)
binary.LittleEndian.PutUint64(buf, d.MaxRepoLabel)
ctx := datastore.NewVersionedCtx(d, v)
store.Put(ctx, maxLabelTKey, buf)
ctx2 := storage.NewDataContext(d, 0)
store.Put(ctx2, maxRepoLabelTKey, buf)
return d.MaxRepoLabel, nil
}
// MigrateInstance migrates a data instance locally from an old storage
// engine to the current configured storage. After completion of the copy,
// the data instance in the old storage is deleted.
func MigrateInstance(uuid dvid.UUID, source dvid.InstanceName, oldStore dvid.Store, c dvid.Config) error {
if manager == nil {
return ErrManagerNotInitialized
}
// Get flatten or not
transmit, _, err := c.GetString("transmit")
if err != nil {
return err
}
var flatten bool
if transmit == "flatten" {
flatten = true
}
// Get the source data instance.
d, err := manager.getDataByUUIDName(uuid, source)
if err != nil {
return err
}
// Get the current store for this data instance.
storer, ok := d.(storage.Accessor)
if !ok {
return fmt.Errorf("unable to migrate data %q: unable to access backing store", d.DataName())
}
curKV, err := storer.GetOrderedKeyValueDB()
if err != nil {
return fmt.Errorf("unable to get backing store for data %q: %v\n", source, err)
}
// Get the old store.
oldKV, ok := oldStore.(storage.OrderedKeyValueDB)
if !ok {
return fmt.Errorf("unable to migrate data %q from store %s which isn't ordered kv store", source, storer)
}
// Abort if the two stores are the same.
if curKV == oldKV {
return fmt.Errorf("old store for data %q seems same as current store", source)
}
// Migrate data asynchronously.
go func() {
if err := copyData(oldKV, curKV, d, nil, uuid, nil, flatten); err != nil {
dvid.Errorf("error in migration of data %q: %v\n", source, err)
return
}
// delete data off old store.
dvid.Infof("Starting delete of instance %q from old storage %q\n", d.DataName(), oldKV)
ctx := storage.NewDataContext(d, 0)
if err := oldKV.DeleteAll(ctx, true); err != nil {
dvid.Errorf("deleting instance %q from %q after copy to %q: %v\n", d.DataName(), oldKV, curKV, err)
return
}
}()
dvid.Infof("Migrating data %q from store %q to store %q ...\n", d.DataName(), oldKV, curKV)
return nil
}
// LoadMutable loads mutable properties of label volumes like the maximum labels
// for each version. Note that we load these max labels from key-value pairs
// rather than data instance properties persistence, because in the case of a crash,
// the actually stored repo data structure may be out-of-date compared to the guaranteed
// up-to-date key-value pairs for max labels.
func (d *Data) LoadMutable(root dvid.VersionID, storedVersion, expectedVersion uint64) (bool, error) {
ctx := storage.NewDataContext(d, 0)
store, err := storage.SmallDataStore()
if err != nil {
return false, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
}
wg := new(sync.WaitGroup)
wg.Add(1)
ch := make(chan *storage.KeyValue)
// Start appropriate migration function if any.
var saveRequired bool
switch storedVersion {
case 0:
// Need to update all max labels and set repo-level max label.
saveRequired = true
go d.migrateMaxLabels(root, wg, ch)
default:
// Load in each version max label without migration.
go d.loadMaxLabels(wg, ch)
// Load in the repo-wide max label.
data, err := store.Get(ctx, maxRepoLabelTKey)
if err != nil {
return false, err
}
d.MaxRepoLabel = binary.LittleEndian.Uint64(data)
}
// Send the max label data per version
minKey, err := ctx.MinVersionKey(maxLabelTKey)
if err != nil {
return false, err
}
maxKey, err := ctx.MaxVersionKey(maxLabelTKey)
if err != nil {
return false, err
}
keysOnly := false
if err = store.SendRange(minKey, maxKey, keysOnly, ch); err != nil {
return false, err
}
wg.Wait()
dvid.Infof("Loaded max label values for labelvol %q with repo-wide max %d\n", d.DataName(), d.MaxRepoLabel)
return saveRequired, nil
}
func (d *Data) loadMaxLabels(wg *sync.WaitGroup, ch chan *storage.KeyValue) {
ctx := storage.NewDataContext(d, 0)
var repoMax uint64
d.MaxLabel = make(map[dvid.VersionID]uint64)
for {
kv := <-ch
if kv == nil {
break
}
v, err := ctx.VersionFromKey(kv.K)
if err != nil {
dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName())
continue
}
if len(kv.V) != 8 {
dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V)
continue
}
label := binary.LittleEndian.Uint64(kv.V)
d.MaxLabel[v] = label
if label > repoMax {
repoMax = label
}
}
// Load in the repo-wide max label.
store, err := storage.MutableStore()
if err != nil {
dvid.Errorf("Data type labelvol had error initializing store: %v\n", err)
return
}
data, err := store.Get(ctx, maxRepoLabelTKey)
if err != nil {
dvid.Errorf("Error getting repo-wide max label: %v\n", err)
return
}
if data == nil || len(data) != 8 {
dvid.Errorf("Could not load repo-wide max label for instance %q. Only got %d bytes, not 64-bit label.\n", d.DataName(), len(data))
dvid.Errorf("Using max label across versions: %d\n", repoMax)
d.MaxRepoLabel = repoMax
} else {
d.MaxRepoLabel = binary.LittleEndian.Uint64(data)
if d.MaxRepoLabel < repoMax {
dvid.Errorf("Saved repo-wide max for instance %q was %d, changed to largest version max %d\n", d.DataName(), d.MaxRepoLabel, repoMax)
d.MaxRepoLabel = repoMax
}
}
wg.Done()
}
func (d *Data) migrateMaxLabels(root dvid.VersionID, wg *sync.WaitGroup, ch chan *storage.KeyValue) {
ctx := storage.NewDataContext(d, 0)
store, err := storage.SmallDataStore()
if err != nil {
dvid.Errorf("Can't initializing small data store: %v\n", err)
}
var maxRepoLabel uint64
d.MaxLabel = make(map[dvid.VersionID]uint64)
for {
kv := <-ch
if kv == nil {
break
}
v, err := ctx.VersionFromKey(kv.K)
if err != nil {
dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName())
continue
}
if len(kv.V) != 8 {
dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V)
continue
}
label := binary.LittleEndian.Uint64(kv.V)
d.MaxLabel[v] = label
if label > maxRepoLabel {
maxRepoLabel = label
}
}
// Adjust the MaxLabel data to make sure we correct for any case of child max < parent max.
d.adjustMaxLabels(store, root)
// Set the repo-wide max label.
d.MaxRepoLabel = maxRepoLabel
buf := make([]byte, 8)
binary.LittleEndian.PutUint64(buf, maxRepoLabel)
store.Put(ctx, maxRepoLabelTKey, buf)
wg.Done()
return
}
// LoadImages bulk loads images using different techniques if it is a multidimensional
// file like HDF5 or a sequence of PNG/JPG/TIF images.
func (d *Data) LoadImages(v dvid.VersionID, offset dvid.Point, filenames []string) error {
if len(filenames) == 0 {
return nil
}
timedLog := dvid.NewTimeLog()
// We only want one PUT on given version for given data to prevent interleaved
// chunk PUTs that could potentially overwrite slice modifications.
ctx := storage.NewDataContext(d, v)
loadMutex := ctx.Mutex()
loadMutex.Lock()
// Handle cleanup given multiple goroutines still writing data.
load := &bulkLoadInfo{filenames: filenames, versionID: v, offset: offset}
defer func() {
loadMutex.Unlock()
if load.extentChanged.Value() {
err := datastore.SaveDataByVersion(v, d)
if err != nil {
dvid.Errorf("Error in trying to save repo for voxel extent change: %v\n", err)
}
}
}()
// Use different loading techniques if we have a potentially multidimensional HDF5 file
// or many 2d images.
var err error
if dvid.Filename(filenames[0]).HasExtensionPrefix("hdf", "h5") {
err = d.loadHDF(load)
} else {
err = d.loadXYImages(load)
}
if err != nil {
timedLog.Infof("RPC load of %d files had error: %v\n", err)
} else {
timedLog.Infof("RPC load of %d files completed.\n", len(filenames))
}
return err
}
func (d *Data) loadMaxLabels(wg *sync.WaitGroup, ch chan *storage.KeyValue) {
ctx := storage.NewDataContext(d, 0)
d.MaxLabel = make(map[dvid.VersionID]uint64)
for {
kv := <-ch
if kv == nil {
wg.Done()
return
}
v, err := ctx.VersionFromKey(kv.K)
if err != nil {
dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName())
continue
}
if len(kv.V) != 8 {
dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V)
continue
}
label := binary.LittleEndian.Uint64(kv.V)
d.MaxLabel[v] = label
}
}
// Given a stored label, make sure our max label tracking is updated.
func (d *Data) casMaxLabel(batch storage.Batch, v dvid.VersionID, label uint64) {
d.ml_mu.Lock()
defer d.ml_mu.Unlock()
save := false
maxLabel, found := d.MaxLabel[v]
if !found {
dvid.Errorf("Bad max label of version %d -- none found!\n", v)
maxLabel = 0
}
if maxLabel < label {
maxLabel = label
save = true
}
if save {
buf := make([]byte, 8)
binary.LittleEndian.PutUint64(buf, maxLabel)
batch.Put(maxLabelTKey, buf)
d.MaxLabel[v] = maxLabel
if d.MaxRepoLabel < maxLabel {
d.MaxRepoLabel = maxLabel
ctx := storage.NewDataContext(d, 0)
store, err := storage.SmallDataStore()
if err != nil {
dvid.Errorf("Data type labelvol had error initializing store: %v\n", err)
} else {
store.Put(ctx, maxRepoLabelTKey, buf)
}
}
}
if err := batch.Commit(); err != nil {
dvid.Errorf("batch put: %v\n", err)
return
}
}
func NewVersionedCtx(data dvid.Data, versionID dvid.VersionID) *VersionedCtx {
return &VersionedCtx{storage.NewDataContext(data, versionID)}
}