// NewDataService returns a new Data instance that fulfills the DataService interface.
// The UUID passed in corresponds to the root UUID of the DAG subgraph that should hold the data.
// This returned Data struct is usually embedded by datatype-specific data instances.
// By default, LZ4 and the default checksum is used.
func NewDataService(t TypeService, rootUUID dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
if _, reserved := reservedNames[string(name)]; reserved {
return nil, fmt.Errorf("cannot use reserved name %q", name)
}
// // Don't allow identical names to be used in the same repo.
// d, err := GetDataByUUIDName(rootUUID, name)
// if err == nil && d != nil {
// return nil, fmt.Errorf("cannot create data instance %q when one already exists in repo with UUID %s", name, rootUUID)
// }
// See if a store was defined for a particular data instance.
store, err := storage.GetAssignedStore(name, rootUUID, t.GetTypeName())
if err != nil {
return nil, err
}
// Make sure we generate a valid UUID for the data instance.
dataUUID := dvid.NewUUID()
if dataUUID == dvid.NilUUID {
return nil, fmt.Errorf("Unable to generate new UUID for data %q creation", name)
}
// Setup the basic data instance structure.
compression, _ := dvid.NewCompression(dvid.LZ4, dvid.DefaultCompression)
data := &Data{
typename: t.GetTypeName(),
typeurl: t.GetTypeURL(),
typeversion: t.GetTypeVersion(),
dataUUID: dataUUID,
id: id,
name: name,
rootUUID: rootUUID,
compression: compression,
checksum: dvid.DefaultChecksum,
syncNames: []dvid.InstanceName{},
syncData: dvid.UUIDSet{},
unversioned: false,
store: store,
}
return data, data.ModifyConfig(c)
}
func (p *pusher) startData(d *DataTxInit) error {
p.stats = new(txStats)
p.stats.lastTime = time.Now()
p.stats.lastBytes = 0
p.dname = d.DataName
// Get the store associated with this data instance.
store, err := storage.GetAssignedStore(d.DataName, p.uuid, d.TypeName)
if err != nil {
return err
}
var ok bool
p.store, ok = store.(storage.KeyValueDB)
if !ok {
return fmt.Errorf("backend store %q for data type %q of tx data %q is not KeyValueDB-compatable", p.store, d.TypeName, d.DataName)
}
dvid.Debugf("Push (session %d) starting transfer of data %q...\n", p.sessionID, d.DataName)
return nil
}
func (p *pusher) readRepo(m *repoTxMsg) (map[dvid.VersionID]struct{}, error) {
dvid.Debugf("Reading repo for push of %s...\n", m.UUID)
if manager == nil {
return nil, ErrManagerNotInitialized
}
p.received += uint64(len(m.Repo))
// Get the repo metadata
p.repo = new(repoT)
if err := p.repo.GobDecode(m.Repo); err != nil {
return nil, err
}
// TODO: Check to see if the transmitted repo is in current metadata.
p.uuid = m.UUID
remoteV, err := p.repo.versionFromUUID(m.UUID) // do this before we remap the repo's IDs
if err != nil {
return nil, err
}
repoID, err := manager.newRepoID()
if err != nil {
return nil, err
}
p.repo.id = repoID
p.instanceMap, p.versionMap, err = p.repo.remapLocalIDs()
if err != nil {
return nil, err
}
// After getting remote repo, adjust data instances for local settings.
for _, d := range p.repo.data {
// see if it needs to adjust versions.
dv, needsUpdate := d.(VersionRemapper)
if needsUpdate {
if err := dv.RemapVersions(p.versionMap); err != nil {
return nil, err
}
}
// check if we have an assigned store for this data instance.
store, err := storage.GetAssignedStore(d.DataName(), d.RootUUID(), d.TypeName())
if err != nil {
return nil, err
}
d.SetBackendStore(store)
dvid.Debugf("Assigning as default store of data instance %q @ %s: %s\n", d.DataName(), d.RootUUID(), store)
}
var versions map[dvid.VersionID]struct{}
switch m.Transmit {
case rpc.TransmitFlatten:
versions = map[dvid.VersionID]struct{}{
remoteV: struct{}{},
}
// Also have to make sure any data instances are rerooted if the root
// no longer exists.
for name, d := range p.repo.data {
_, found := manager.uuidToVersion[d.RootUUID()]
if !found {
p.repo.data[name].SetRootUUID(m.UUID)
}
}
case rpc.TransmitAll:
versions, err = getDeltaAll(p.repo, m.UUID)
if err != nil {
return nil, err
}
case rpc.TransmitBranch:
versions, err = getDeltaBranch(p.repo, m.UUID)
if err != nil {
return nil, err
}
}
if versions == nil {
return nil, fmt.Errorf("no push required -- remote has necessary versions")
}
dvid.Debugf("Finished comparing repos -- requesting %d versions from source.\n", len(versions))
return versions, nil
}