// NewLabel returns a new label for the given version. func (d *Data) NewLabel(v dvid.VersionID) (uint64, error) { d.ml_mu.Lock() defer d.ml_mu.Unlock() // Make sure we aren't trying to increment a label on a locked node. locked, err := datastore.LockedVersion(v) if err != nil { return 0, err } if locked { return 0, fmt.Errorf("can't ask for new label in a locked version id %d", v) } // Increment and store. d.MaxRepoLabel++ d.MaxLabel[v] = d.MaxRepoLabel store, err := storage.SmallDataStore() if err != nil { return 0, fmt.Errorf("can't initializing small data store: %v\n", err) } buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, d.MaxRepoLabel) ctx := datastore.NewVersionedCtx(d, v) store.Put(ctx, maxLabelTKey, buf) ctx2 := storage.NewDataContext(d, 0) store.Put(ctx2, maxRepoLabelTKey, buf) return d.MaxRepoLabel, nil }
// MigrateInstance migrates a data instance locally from an old storage // engine to the current configured storage. After completion of the copy, // the data instance in the old storage is deleted. func MigrateInstance(uuid dvid.UUID, source dvid.InstanceName, oldStore dvid.Store, c dvid.Config) error { if manager == nil { return ErrManagerNotInitialized } // Get flatten or not transmit, _, err := c.GetString("transmit") if err != nil { return err } var flatten bool if transmit == "flatten" { flatten = true } // Get the source data instance. d, err := manager.getDataByUUIDName(uuid, source) if err != nil { return err } // Get the current store for this data instance. storer, ok := d.(storage.Accessor) if !ok { return fmt.Errorf("unable to migrate data %q: unable to access backing store", d.DataName()) } curKV, err := storer.GetOrderedKeyValueDB() if err != nil { return fmt.Errorf("unable to get backing store for data %q: %v\n", source, err) } // Get the old store. oldKV, ok := oldStore.(storage.OrderedKeyValueDB) if !ok { return fmt.Errorf("unable to migrate data %q from store %s which isn't ordered kv store", source, storer) } // Abort if the two stores are the same. if curKV == oldKV { return fmt.Errorf("old store for data %q seems same as current store", source) } // Migrate data asynchronously. go func() { if err := copyData(oldKV, curKV, d, nil, uuid, nil, flatten); err != nil { dvid.Errorf("error in migration of data %q: %v\n", source, err) return } // delete data off old store. dvid.Infof("Starting delete of instance %q from old storage %q\n", d.DataName(), oldKV) ctx := storage.NewDataContext(d, 0) if err := oldKV.DeleteAll(ctx, true); err != nil { dvid.Errorf("deleting instance %q from %q after copy to %q: %v\n", d.DataName(), oldKV, curKV, err) return } }() dvid.Infof("Migrating data %q from store %q to store %q ...\n", d.DataName(), oldKV, curKV) return nil }
// LoadMutable loads mutable properties of label volumes like the maximum labels // for each version. Note that we load these max labels from key-value pairs // rather than data instance properties persistence, because in the case of a crash, // the actually stored repo data structure may be out-of-date compared to the guaranteed // up-to-date key-value pairs for max labels. func (d *Data) LoadMutable(root dvid.VersionID, storedVersion, expectedVersion uint64) (bool, error) { ctx := storage.NewDataContext(d, 0) store, err := storage.SmallDataStore() if err != nil { return false, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) } wg := new(sync.WaitGroup) wg.Add(1) ch := make(chan *storage.KeyValue) // Start appropriate migration function if any. var saveRequired bool switch storedVersion { case 0: // Need to update all max labels and set repo-level max label. saveRequired = true go d.migrateMaxLabels(root, wg, ch) default: // Load in each version max label without migration. go d.loadMaxLabels(wg, ch) // Load in the repo-wide max label. data, err := store.Get(ctx, maxRepoLabelTKey) if err != nil { return false, err } d.MaxRepoLabel = binary.LittleEndian.Uint64(data) } // Send the max label data per version minKey, err := ctx.MinVersionKey(maxLabelTKey) if err != nil { return false, err } maxKey, err := ctx.MaxVersionKey(maxLabelTKey) if err != nil { return false, err } keysOnly := false if err = store.SendRange(minKey, maxKey, keysOnly, ch); err != nil { return false, err } wg.Wait() dvid.Infof("Loaded max label values for labelvol %q with repo-wide max %d\n", d.DataName(), d.MaxRepoLabel) return saveRequired, nil }
func (d *Data) loadMaxLabels(wg *sync.WaitGroup, ch chan *storage.KeyValue) { ctx := storage.NewDataContext(d, 0) var repoMax uint64 d.MaxLabel = make(map[dvid.VersionID]uint64) for { kv := <-ch if kv == nil { break } v, err := ctx.VersionFromKey(kv.K) if err != nil { dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName()) continue } if len(kv.V) != 8 { dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V) continue } label := binary.LittleEndian.Uint64(kv.V) d.MaxLabel[v] = label if label > repoMax { repoMax = label } } // Load in the repo-wide max label. store, err := storage.MutableStore() if err != nil { dvid.Errorf("Data type labelvol had error initializing store: %v\n", err) return } data, err := store.Get(ctx, maxRepoLabelTKey) if err != nil { dvid.Errorf("Error getting repo-wide max label: %v\n", err) return } if data == nil || len(data) != 8 { dvid.Errorf("Could not load repo-wide max label for instance %q. Only got %d bytes, not 64-bit label.\n", d.DataName(), len(data)) dvid.Errorf("Using max label across versions: %d\n", repoMax) d.MaxRepoLabel = repoMax } else { d.MaxRepoLabel = binary.LittleEndian.Uint64(data) if d.MaxRepoLabel < repoMax { dvid.Errorf("Saved repo-wide max for instance %q was %d, changed to largest version max %d\n", d.DataName(), d.MaxRepoLabel, repoMax) d.MaxRepoLabel = repoMax } } wg.Done() }
func (d *Data) migrateMaxLabels(root dvid.VersionID, wg *sync.WaitGroup, ch chan *storage.KeyValue) { ctx := storage.NewDataContext(d, 0) store, err := storage.SmallDataStore() if err != nil { dvid.Errorf("Can't initializing small data store: %v\n", err) } var maxRepoLabel uint64 d.MaxLabel = make(map[dvid.VersionID]uint64) for { kv := <-ch if kv == nil { break } v, err := ctx.VersionFromKey(kv.K) if err != nil { dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName()) continue } if len(kv.V) != 8 { dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V) continue } label := binary.LittleEndian.Uint64(kv.V) d.MaxLabel[v] = label if label > maxRepoLabel { maxRepoLabel = label } } // Adjust the MaxLabel data to make sure we correct for any case of child max < parent max. d.adjustMaxLabels(store, root) // Set the repo-wide max label. d.MaxRepoLabel = maxRepoLabel buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, maxRepoLabel) store.Put(ctx, maxRepoLabelTKey, buf) wg.Done() return }
// LoadImages bulk loads images using different techniques if it is a multidimensional // file like HDF5 or a sequence of PNG/JPG/TIF images. func (d *Data) LoadImages(v dvid.VersionID, offset dvid.Point, filenames []string) error { if len(filenames) == 0 { return nil } timedLog := dvid.NewTimeLog() // We only want one PUT on given version for given data to prevent interleaved // chunk PUTs that could potentially overwrite slice modifications. ctx := storage.NewDataContext(d, v) loadMutex := ctx.Mutex() loadMutex.Lock() // Handle cleanup given multiple goroutines still writing data. load := &bulkLoadInfo{filenames: filenames, versionID: v, offset: offset} defer func() { loadMutex.Unlock() if load.extentChanged.Value() { err := datastore.SaveDataByVersion(v, d) if err != nil { dvid.Errorf("Error in trying to save repo for voxel extent change: %v\n", err) } } }() // Use different loading techniques if we have a potentially multidimensional HDF5 file // or many 2d images. var err error if dvid.Filename(filenames[0]).HasExtensionPrefix("hdf", "h5") { err = d.loadHDF(load) } else { err = d.loadXYImages(load) } if err != nil { timedLog.Infof("RPC load of %d files had error: %v\n", err) } else { timedLog.Infof("RPC load of %d files completed.\n", len(filenames)) } return err }
func (d *Data) loadMaxLabels(wg *sync.WaitGroup, ch chan *storage.KeyValue) { ctx := storage.NewDataContext(d, 0) d.MaxLabel = make(map[dvid.VersionID]uint64) for { kv := <-ch if kv == nil { wg.Done() return } v, err := ctx.VersionFromKey(kv.K) if err != nil { dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName()) continue } if len(kv.V) != 8 { dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V) continue } label := binary.LittleEndian.Uint64(kv.V) d.MaxLabel[v] = label } }
// Given a stored label, make sure our max label tracking is updated. func (d *Data) casMaxLabel(batch storage.Batch, v dvid.VersionID, label uint64) { d.ml_mu.Lock() defer d.ml_mu.Unlock() save := false maxLabel, found := d.MaxLabel[v] if !found { dvid.Errorf("Bad max label of version %d -- none found!\n", v) maxLabel = 0 } if maxLabel < label { maxLabel = label save = true } if save { buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, maxLabel) batch.Put(maxLabelTKey, buf) d.MaxLabel[v] = maxLabel if d.MaxRepoLabel < maxLabel { d.MaxRepoLabel = maxLabel ctx := storage.NewDataContext(d, 0) store, err := storage.SmallDataStore() if err != nil { dvid.Errorf("Data type labelvol had error initializing store: %v\n", err) } else { store.Put(ctx, maxRepoLabelTKey, buf) } } } if err := batch.Commit(); err != nil { dvid.Errorf("batch put: %v\n", err) return } }
func NewVersionedCtx(data dvid.Data, versionID dvid.VersionID) *VersionedCtx { return &VersionedCtx{storage.NewDataContext(data, versionID)} }