func (n *nodeRunner) run() error {
var err error
for name, parentChan := range n.parentChans {
protolog.Debug(&NodeWaiting{Node: n.nodeName, ParentNode: name})
select {
case parentErr := <-parentChan:
if parentErr != nil {
err = parentErr
}
continue
case <-n.cancel:
return err
}
}
protolog.Debug(&NodeFinishedWaiting{Node: n.nodeName, ParentError: errorString(err)})
if err == nil {
protolog.Info(&NodeStarting{Node: n.nodeName})
err = n.f()
protolog.Info(&NodeFinished{Node: n.nodeName, Error: errorString(err)})
}
for name, childChan := range n.childrenChans {
protolog.Debug(&NodeSending{Node: n.nodeName, ChildNode: name, Error: errorString(err)})
childChan <- err
close(childChan)
}
return err
}
func (l *logger) LogMethodInvocation(methodInvocation *MethodInvocation) {
if methodInvocation.Error != "" {
protolog.Error(methodInvocation)
} else {
protolog.Info(methodInvocation)
}
}
func execTransID(path string) (result string, retErr error) {
defer func() {
protolog.Info(&TransID{path, result, errorToString(retErr)})
}()
// "9223372036854775810" == 2 ** 63 we use a very big number there so that
// we get the transid of the from path. According to the internet this is
// the nicest way to get it from btrfs.
var buffer bytes.Buffer
if err := executil.RunStdout(&buffer, "btrfs", "subvolume", "find-new", path, "9223372036854775808"); err != nil {
return "", err
}
scanner := bufio.NewScanner(&buffer)
for scanner.Scan() {
// scanner.Text() looks like this:
// transid marker was 907
// 0 1 2 3
tokens := strings.Split(scanner.Text(), " ")
if len(tokens) != 4 {
return "", fmt.Errorf("pachyderm: failed to parse find-new output")
}
return tokens[3], nil
}
if scanner.Err() != nil {
return "", scanner.Err()
}
return "", fmt.Errorf("pachyderm: empty output from find-new")
}
func (f *file) Open(ctx context.Context, request *fuse.OpenRequest, response *fuse.OpenResponse) (_ fs.Handle, retErr error) {
defer func() {
protolog.Info(&FileRead{&f.Node, errorToString(retErr)})
}()
atomic.AddInt32(&f.handles, 1)
return f, nil
}
func (a *discoveryAddresser) announceState(
id string,
address string,
server Server,
versionChan chan int64,
cancel chan bool,
) error {
serverState := &proto.ServerState{
Id: id,
Address: address,
Version: InvalidVersion,
}
for {
shards, err := server.LocalShards()
if err != nil {
return err
}
serverState.Shards = shards
encodedServerState, err := marshaler.MarshalToString(serverState)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.serverStateKey(id), encodedServerState, holdTTL); err != nil {
return err
}
protolog.Info(&log.SetServerState{serverState})
select {
case <-cancel:
return nil
case version := <-versionChan:
serverState.Version = version
case <-time.After(time.Second * time.Duration(holdTTL/2)):
}
}
}
func execSend(path string, parent string, diff io.Writer) (retErr error) {
defer func() {
protolog.Info(&Send{path, parent, errorToString(retErr)})
}()
if parent == "" {
return executil.RunStdout(diff, "btrfs", "send", path)
}
return executil.RunStdout(diff, "btrfs", "send", "-p", parent, path)
}
func execSubvolumeExists(path string) (result bool) {
defer func() {
protolog.Info(&SubvolumeExists{path, result})
}()
if err := executil.Run("btrfs", "subvolume", "show", path); err != nil {
return false
}
return true
}
func execSubvolumeSnapshot(src string, dest string, readOnly bool) (retErr error) {
defer func() {
protolog.Info(&SubvolumeSnapshot{src, dest, readOnly, errorToString(retErr)})
}()
if readOnly {
return pkgexec.Run("btrfs", "subvolume", "snapshot", "-r", src, dest)
}
return pkgexec.Run("btrfs", "subvolume", "snapshot", src, dest)
}
func (a *discoveryAddresser) Register(cancel chan bool, id string, address string, server Server) (retErr error) {
protolog.Info(&log.StartRegister{id, address})
defer func() {
protolog.Info(&log.FinishRegister{id, address, errorToString(retErr)})
}()
var once sync.Once
versionChan := make(chan int64)
internalCancel := make(chan bool)
var wg sync.WaitGroup
wg.Add(3)
go func() {
defer wg.Done()
if err := a.announceState(id, address, server, versionChan, internalCancel); err != nil {
once.Do(func() {
retErr = err
close(internalCancel)
})
}
}()
go func() {
defer wg.Done()
if err := a.fillRoles(id, server, versionChan, internalCancel); err != nil {
once.Do(func() {
retErr = err
close(internalCancel)
})
}
}()
go func() {
defer wg.Done()
select {
case <-cancel:
once.Do(func() {
retErr = ErrCancelled
close(internalCancel)
})
case <-internalCancel:
}
}()
wg.Wait()
return
}
func (d *directory) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
defer func() {
protolog.Info(&DirectoryAttr{&d.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
}()
if d.Write {
a.Mode = os.ModeDir | 0775
} else {
a.Mode = os.ModeDir | 0555
}
return nil
}
func (f *file) Read(ctx context.Context, request *fuse.ReadRequest, response *fuse.ReadResponse) (retErr error) {
defer func() {
protolog.Info(&FileRead{&f.Node, errorToString(retErr)})
}()
buffer := bytes.NewBuffer(make([]byte, 0, request.Size))
if err := pfsutil.GetFile(f.fs.apiClient, f.RepoName, f.CommitID, f.Path, request.Offset, int64(request.Size), buffer); err != nil {
return err
}
response.Data = buffer.Bytes()
return nil
}
func (d *directory) Lookup(ctx context.Context, name string) (result fs.Node, retErr error) {
defer func() {
protolog.Info(&DirectoryLookup{&d.Node, name, getNode(result), errorToString(retErr)})
}()
if d.File.Commit.Repo.Name == "" {
return d.lookUpRepo(ctx, name)
}
if d.File.Commit.Id == "" {
return d.lookUpCommit(ctx, name)
}
return d.lookUpFile(ctx, name)
}
func execSubvolumeFindNew(commit string, fromCommit string, out io.Writer) (retErr error) {
defer func() {
protolog.Info(&SubvolumeFindNew{commit, fromCommit, errorToString(retErr)})
}()
if fromCommit == "" {
return executil.RunStdout(out, "btrfs", "subvolume", "find-new", commit, "0")
}
transid, err := execTransID(fromCommit)
if err != nil {
return err
}
return executil.RunStdout(out, "btrfs", "subvolume", "find-new", commit, transid)
}
func (d *directory) Mkdir(ctx context.Context, request *fuse.MkdirRequest) (result fs.Node, retErr error) {
defer func() {
protolog.Info(&DirectoryMkdir{&d.Node, getNode(result), errorToString(retErr)})
}()
if d.File.Commit.Id == "" {
return nil, fuse.EPERM
}
if err := pfsutil.MakeDirectory(d.fs.apiClient, d.File.Commit.Repo.Name, d.File.Commit.Id, path.Join(d.File.Path, request.Name)); err != nil {
return nil, err
}
localResult := d.copy()
localResult.File.Path = path.Join(localResult.File.Path, request.Name)
return localResult, nil
}
func (f *filesystem) Root() (result fs.Node, retErr error) {
defer func() {
protolog.Info(&Root{&f.Filesystem, getNode(result), errorToString(retErr)})
}()
return &directory{
f,
Node{&pfs.File{
Commit: &pfs.Commit{
Repo: &pfs.Repo{},
},
},
true},
}, nil
}
func (a *discoveryAddresser) GetShardToMasterAddress(version int64) (result map[uint64]string, retErr error) {
defer func() {
protolog.Info(&log.GetShardToMasterAddress{version, result, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return nil, err
}
_result := make(map[uint64]string)
for shard, shardAddresses := range addresses.Addresses {
_result[shard] = shardAddresses.Master
}
return _result, nil
}
func (a *discoveryAddresser) GetReplicaAddresses(shard uint64, version int64) (result map[string]bool, retErr error) {
defer func() {
protolog.Info(&log.GetReplicaAddresses{shard, version, result, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return nil, err
}
shardAddresses, ok := addresses.Addresses[shard]
if !ok {
return nil, fmt.Errorf("shard %d not found", shard)
}
return shardAddresses.Replicas, nil
}
func (a *discoveryAddresser) GetMasterAddress(shard uint64, version int64) (result string, ok bool, retErr error) {
defer func() {
protolog.Info(&log.GetMasterAddress{shard, version, result, ok, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return "", false, err
}
shardAddresses, ok := addresses.Addresses[shard]
if !ok {
return "", false, nil
}
return shardAddresses.Master, true, nil
}
func (f *file) Write(ctx context.Context, request *fuse.WriteRequest, response *fuse.WriteResponse) (retErr error) {
defer func() {
protolog.Info(&FileWrite{&f.Node, errorToString(retErr)})
}()
written, err := pfsutil.PutFile(f.fs.apiClient, f.File.Commit.Repo.Name, f.File.Commit.Id, f.File.Path, request.Offset, bytes.NewReader(request.Data))
if err != nil {
return err
}
response.Size = written
if f.size < request.Offset+int64(written) {
f.size = request.Offset + int64(written)
}
return nil
}
func (a *apiServer) CreatePipelineRun(ctx context.Context, request *pps.CreatePipelineRunRequest) (*pps.PipelineRun, error) {
pipelineRun := &pps.PipelineRun{
Id: strings.Replace(uuid.NewV4().String(), "-", "", -1),
PipelineId: request.PipelineId,
}
if err := a.storeClient.CreatePipelineRun(pipelineRun); err != nil {
return nil, err
}
protolog.Info(
&CreatedPipelineRun{
PipelineRun: pipelineRun,
},
)
return pipelineRun, nil
}
func (d *directory) Create(ctx context.Context, request *fuse.CreateRequest, response *fuse.CreateResponse) (result fs.Node, _ fs.Handle, retErr error) {
defer func() {
protolog.Info(&DirectoryCreate{&d.Node, getNode(result), errorToString(retErr)})
}()
if d.File.Commit.Id == "" {
return nil, 0, fuse.EPERM
}
directory := d.copy()
directory.File.Path = path.Join(directory.File.Path, request.Name)
localResult := &file{*directory, 0, 0}
handle, err := localResult.Open(ctx, nil, nil)
if err != nil {
return nil, nil, err
}
return localResult, handle, nil
}
func (d *directory) ReadDirAll(ctx context.Context) (result []fuse.Dirent, retErr error) {
defer func() {
var dirents []*Dirent
for _, dirent := range result {
dirents = append(dirents, &Dirent{dirent.Inode, dirent.Name})
}
protolog.Info(&DirectoryReadDirAll{&d.Node, dirents, errorToString(retErr)})
}()
if d.File.Commit.Repo.Name == "" {
return d.readRepos(ctx)
}
if d.File.Commit.Id == "" {
return d.readCommits(ctx)
}
return d.readFiles(ctx)
}
func (a *apiServer) CreatePipelineRun(ctx context.Context, request *pps.CreatePipelineRunRequest) (*pps.PipelineRun, error) {
pipelineRun := &pps.PipelineRun{
Id: strings.Replace(uuid.NewV4().String(), "-", "", -1),
PipelineId: request.PipelineId,
}
// TODO(pedge): should be transactional with call to CreatePipelineRunStatus
if err := a.storeClient.CreatePipelineRun(pipelineRun); err != nil {
return nil, err
}
if err := a.storeClient.CreatePipelineRunStatus(pipelineRun.Id, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED); err != nil {
return nil, err
}
protolog.Info(
&CreatedPipelineRun{
PipelineRun: pipelineRun,
},
)
return pipelineRun, nil
}
func (f *file) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
defer func() {
protolog.Info(&FileAttr{&f.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
}()
fileInfo, err := pfsutil.InspectFile(
f.fs.apiClient,
f.RepoName,
f.CommitID,
f.Path,
)
if err != nil {
return err
}
if fileInfo != nil {
a.Size = fileInfo.SizeBytes
}
a.Mode = 0666
return nil
}
func (a *discoveryAddresser) GetShardToReplicaAddresses(version int64) (result map[uint64]map[string]bool, retErr error) {
defer func() {
// We need resultPrime is because proto3 can't do maps of maps.
resultPrime := make(map[uint64]*log.ReplicaAddresses)
for shard, addresses := range result {
resultPrime[shard] = &log.ReplicaAddresses{addresses}
}
protolog.Info(&log.GetShardToReplicaAddresses{version, resultPrime, errorToString(retErr)})
}()
addresses, err := a.getAddresses(version)
if err != nil {
return nil, err
}
_result := make(map[uint64]map[string]bool)
for shard, shardAddresses := range addresses.Addresses {
_result[shard] = shardAddresses.Replicas
}
return _result, nil
}
func (a *discoveryAddresser) Version() (result int64, retErr error) {
defer func() {
protolog.Info(&log.Version{result, errorToString(retErr)})
}()
minVersion := int64(math.MaxInt64)
encodedServerStates, err := a.discoveryClient.GetAll(a.serverStateDir())
if err != nil {
return 0, err
}
for _, encodedServerState := range encodedServerStates {
serverState, err := decodeServerState(encodedServerState)
if err != nil {
return 0, err
}
if serverState.Version < minVersion {
minVersion = serverState.Version
}
}
return minVersion, nil
}
func (f *file) Read(ctx context.Context, request *fuse.ReadRequest, response *fuse.ReadResponse) (retErr error) {
defer func() {
protolog.Info(&FileRead{&f.Node, errorToString(retErr)})
}()
var buffer bytes.Buffer
if err := pfsutil.GetFile(
f.fs.apiClient,
f.File.Commit.Repo.Name,
f.File.Commit.Id,
f.File.Path,
request.Offset,
int64(request.Size),
f.fs.Shard,
&buffer,
); err != nil {
return err
}
response.Data = buffer.Bytes()
return nil
}
func execSubvolumeList(path string, fromCommit string, ascending bool, out io.Writer) (retErr error) {
defer func() {
protolog.Info(&SubvolumeList{path, fromCommit, ascending, errorToString(retErr)})
}()
var sort string
if ascending {
sort = "+ogen"
} else {
sort = "-ogen"
}
if fromCommit == "" {
return executil.RunStdout(out, "btrfs", "subvolume", "list", "-a", "--sort", sort, path)
}
transid, err := execTransID(fromCommit)
if err != nil {
return err
}
return executil.RunStdout(out, "btrfs", "subvolume", "list", "-aC", "+"+transid, "--sort", sort, path)
}
func (a *discoveryAddresser) fillRoles(
id string,
server Server,
versionChan chan int64,
cancel chan bool,
) error {
oldRoles := make(map[int64]proto.ServerRole)
return a.discoveryClient.WatchAll(
a.serverRoleKey(id),
cancel,
func(encodedServerRoles map[string]string) error {
roles := make(map[int64]proto.ServerRole)
var versions int64Slice
// Decode the roles
for _, encodedServerRole := range encodedServerRoles {
var serverRole proto.ServerRole
if err := jsonpb.UnmarshalString(encodedServerRole, &serverRole); err != nil {
return err
}
roles[serverRole.Version] = serverRole
versions = append(versions, serverRole.Version)
}
sort.Sort(versions)
if len(versions) > 2 {
versions = versions[0:2]
}
// For each new version bring the server up to date
for _, version := range versions {
if _, ok := oldRoles[version]; ok {
// we've already seen these roles, so nothing to do here
continue
}
serverRole := roles[version]
var wg sync.WaitGroup
var addShardErr error
for _, shard := range shards(serverRole) {
if !containsShard(oldRoles, shard) {
wg.Add(1)
shard := shard
go func() {
defer wg.Done()
if err := server.AddShard(shard); err != nil && addShardErr == nil {
addShardErr = err
}
}()
}
}
wg.Wait()
if addShardErr != nil {
protolog.Info(&log.AddServerRole{&serverRole, addShardErr.Error()})
return addShardErr
}
protolog.Info(&log.AddServerRole{&serverRole, ""})
oldRoles[version] = serverRole
versionChan <- version
}
// See if there are any old roles that aren't needed
for version, serverRole := range oldRoles {
var wg sync.WaitGroup
var removeShardErr error
if _, ok := roles[version]; ok {
// these roles haven't expired yet, so nothing to do
continue
}
for _, shard := range shards(serverRole) {
if !containsShard(roles, shard) {
wg.Add(1)
shard := shard
go func(shard uint64) {
defer wg.Done()
if err := server.RemoveShard(shard); err != nil && removeShardErr == nil {
removeShardErr = err
}
}(shard)
}
}
wg.Wait()
if removeShardErr != nil {
protolog.Info(&log.RemoveServerRole{&serverRole, removeShardErr.Error()})
return removeShardErr
}
protolog.Info(&log.RemoveServerRole{&serverRole, ""})
}
oldRoles = make(map[int64]proto.ServerRole)
for _, version := range versions {
oldRoles[version] = roles[version]
}
return nil
},
)
}
func (a *discoveryAddresser) AssignRoles(cancel chan bool) (retErr error) {
protolog.Info(&log.StartAssignRoles{})
defer func() {
protolog.Info(&log.FinishAssignRoles{errorToString(retErr)})
}()
var version int64
oldServers := make(map[string]bool)
oldRoles := make(map[string]*proto.ServerRole)
oldMasters := make(map[uint64]string)
oldReplicas := make(map[uint64][]string)
var oldMinVersion int64
// Reconstruct state from a previous run
serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
if err != nil {
return err
}
for _, encodedServerRole := range serverRoles {
serverRole, err := decodeServerRole(encodedServerRole)
if err != nil {
return err
}
if oldServerRole, ok := oldRoles[serverRole.Id]; !ok || oldServerRole.Version < serverRole.Version {
oldRoles[serverRole.Id] = serverRole
oldServers[serverRole.Id] = true
}
if version < serverRole.Version+1 {
version = serverRole.Version + 1
}
}
for _, oldServerRole := range oldRoles {
for shard := range oldServerRole.Masters {
oldMasters[shard] = oldServerRole.Id
}
for shard := range oldServerRole.Replicas {
oldReplicas[shard] = append(oldReplicas[shard], oldServerRole.Id)
}
}
err = a.discoveryClient.WatchAll(a.serverStateDir(), cancel,
func(encodedServerStates map[string]string) error {
if len(encodedServerStates) == 0 {
return nil
}
newServerStates := make(map[string]*proto.ServerState)
shardLocations := make(map[uint64][]string)
newRoles := make(map[string]*proto.ServerRole)
newMasters := make(map[uint64]string)
newReplicas := make(map[uint64][]string)
masterRolesPerServer := a.sharder.NumShards() / uint64(len(encodedServerStates))
masterRolesRemainder := a.sharder.NumShards() % uint64(len(encodedServerStates))
replicaRolesPerServer := (a.sharder.NumShards() * (a.sharder.NumReplicas())) / uint64(len(encodedServerStates))
replicaRolesRemainder := (a.sharder.NumShards() * (a.sharder.NumReplicas())) % uint64(len(encodedServerStates))
for _, encodedServerState := range encodedServerStates {
serverState, err := decodeServerState(encodedServerState)
if err != nil {
return err
}
newServerStates[serverState.Id] = serverState
newRoles[serverState.Id] = &proto.ServerRole{
Id: serverState.Id,
Version: version,
Masters: make(map[uint64]bool),
Replicas: make(map[uint64]bool),
}
for shard := range serverState.Shards {
shardLocations[shard] = append(shardLocations[shard], serverState.Id)
}
}
// See if there's any roles we can delete
minVersion := int64(math.MaxInt64)
for _, serverState := range newServerStates {
if serverState.Version < minVersion {
minVersion = serverState.Version
}
}
// Delete roles that no servers are using anymore
if minVersion > oldMinVersion {
oldMinVersion = minVersion
serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
if err != nil {
return err
}
for key, encodedServerRole := range serverRoles {
serverRole, err := decodeServerRole(encodedServerRole)
if err != nil {
return err
}
if serverRole.Version < minVersion {
if err := a.discoveryClient.Delete(key); err != nil {
return err
}
protolog.Info(&log.DeleteServerRole{serverRole})
}
}
}
// if the servers are identical to last time then we know we'll
// assign shards the same way
if sameServers(oldServers, newServerStates) {
return nil
}
Master:
for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
if id, ok := oldMasters[shard]; ok {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
for _, id := range oldReplicas[shard] {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
for _, id := range shardLocations[shard] {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
for id := range newServerStates {
if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
continue Master
}
}
protolog.Error(&log.FailedToAssignRoles{
ServerStates: newServerStates,
NumShards: a.sharder.NumShards(),
NumReplicas: a.sharder.NumReplicas(),
})
return nil
}
for replica := uint64(0); replica < a.sharder.NumReplicas(); replica++ {
Replica:
for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
if id, ok := oldMasters[shard]; ok {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for _, id := range oldReplicas[shard] {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for _, id := range shardLocations[shard] {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for id := range newServerStates {
if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
continue Replica
}
}
for id := range newServerStates {
if swapReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer) {
continue Replica
}
}
protolog.Error(&log.FailedToAssignRoles{
ServerStates: newServerStates,
NumShards: a.sharder.NumShards(),
NumReplicas: a.sharder.NumReplicas(),
})
return nil
}
}
addresses := proto.Addresses{
Version: version,
Addresses: make(map[uint64]*proto.ShardAddresses),
}
for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
addresses.Addresses[shard] = &proto.ShardAddresses{Replicas: make(map[string]bool)}
}
for id, serverRole := range newRoles {
encodedServerRole, err := marshaler.MarshalToString(serverRole)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.serverRoleKeyVersion(id, version), encodedServerRole, 0); err != nil {
return err
}
protolog.Info(&log.SetServerRole{serverRole})
address := newServerStates[id].Address
for shard := range serverRole.Masters {
shardAddresses := addresses.Addresses[shard]
shardAddresses.Master = address
addresses.Addresses[shard] = shardAddresses
}
for shard := range serverRole.Replicas {
shardAddresses := addresses.Addresses[shard]
shardAddresses.Replicas[address] = true
addresses.Addresses[shard] = shardAddresses
}
}
encodedAddresses, err := marshaler.MarshalToString(&addresses)
if err != nil {
return err
}
if err := a.discoveryClient.Set(a.addressesKey(version), encodedAddresses, 0); err != nil {
return err
}
protolog.Info(&log.SetAddresses{&addresses})
version++
oldServers = make(map[string]bool)
for id := range newServerStates {
oldServers[id] = true
}
oldRoles = newRoles
oldMasters = newMasters
oldReplicas = newReplicas
return nil
})
if err == discovery.ErrCancelled {
return ErrCancelled
}
return err
}