Beispiel #1
0
func (n *nodeRunner) run() error {
	var err error
	for name, parentChan := range n.parentChans {
		protolog.Debug(&NodeWaiting{Node: n.nodeName, ParentNode: name})
		select {
		case parentErr := <-parentChan:
			if parentErr != nil {
				err = parentErr
			}
			continue
		case <-n.cancel:
			return err
		}
	}
	protolog.Debug(&NodeFinishedWaiting{Node: n.nodeName, ParentError: errorString(err)})
	if err == nil {
		protolog.Info(&NodeStarting{Node: n.nodeName})
		err = n.f()
		protolog.Info(&NodeFinished{Node: n.nodeName, Error: errorString(err)})
	}
	for name, childChan := range n.childrenChans {
		protolog.Debug(&NodeSending{Node: n.nodeName, ChildNode: name, Error: errorString(err)})
		childChan <- err
		close(childChan)
	}
	return err
}
Beispiel #2
0
func (l *logger) LogMethodInvocation(methodInvocation *MethodInvocation) {
	if methodInvocation.Error != "" {
		protolog.Error(methodInvocation)
	} else {
		protolog.Info(methodInvocation)
	}
}
Beispiel #3
0
func execTransID(path string) (result string, retErr error) {
	defer func() {
		protolog.Info(&TransID{path, result, errorToString(retErr)})
	}()
	//  "9223372036854775810" == 2 ** 63 we use a very big number there so that
	//  we get the transid of the from path. According to the internet this is
	//  the nicest way to get it from btrfs.
	var buffer bytes.Buffer
	if err := executil.RunStdout(&buffer, "btrfs", "subvolume", "find-new", path, "9223372036854775808"); err != nil {
		return "", err
	}
	scanner := bufio.NewScanner(&buffer)
	for scanner.Scan() {
		// scanner.Text() looks like this:
		// transid marker was 907
		// 0       1      2   3
		tokens := strings.Split(scanner.Text(), " ")
		if len(tokens) != 4 {
			return "", fmt.Errorf("pachyderm: failed to parse find-new output")
		}
		return tokens[3], nil
	}
	if scanner.Err() != nil {
		return "", scanner.Err()
	}
	return "", fmt.Errorf("pachyderm: empty output from find-new")
}
Beispiel #4
0
func (f *file) Open(ctx context.Context, request *fuse.OpenRequest, response *fuse.OpenResponse) (_ fs.Handle, retErr error) {
	defer func() {
		protolog.Info(&FileRead{&f.Node, errorToString(retErr)})
	}()
	atomic.AddInt32(&f.handles, 1)
	return f, nil
}
Beispiel #5
0
func (a *discoveryAddresser) announceState(
	id string,
	address string,
	server Server,
	versionChan chan int64,
	cancel chan bool,
) error {
	serverState := &proto.ServerState{
		Id:      id,
		Address: address,
		Version: InvalidVersion,
	}
	for {
		shards, err := server.LocalShards()
		if err != nil {
			return err
		}
		serverState.Shards = shards
		encodedServerState, err := marshaler.MarshalToString(serverState)
		if err != nil {
			return err
		}
		if err := a.discoveryClient.Set(a.serverStateKey(id), encodedServerState, holdTTL); err != nil {
			return err
		}
		protolog.Info(&log.SetServerState{serverState})
		select {
		case <-cancel:
			return nil
		case version := <-versionChan:
			serverState.Version = version
		case <-time.After(time.Second * time.Duration(holdTTL/2)):
		}
	}
}
Beispiel #6
0
func execSend(path string, parent string, diff io.Writer) (retErr error) {
	defer func() {
		protolog.Info(&Send{path, parent, errorToString(retErr)})
	}()
	if parent == "" {
		return executil.RunStdout(diff, "btrfs", "send", path)
	}
	return executil.RunStdout(diff, "btrfs", "send", "-p", parent, path)
}
Beispiel #7
0
func execSubvolumeExists(path string) (result bool) {
	defer func() {
		protolog.Info(&SubvolumeExists{path, result})
	}()
	if err := executil.Run("btrfs", "subvolume", "show", path); err != nil {
		return false
	}
	return true
}
Beispiel #8
0
func execSubvolumeSnapshot(src string, dest string, readOnly bool) (retErr error) {
	defer func() {
		protolog.Info(&SubvolumeSnapshot{src, dest, readOnly, errorToString(retErr)})
	}()
	if readOnly {
		return pkgexec.Run("btrfs", "subvolume", "snapshot", "-r", src, dest)
	}
	return pkgexec.Run("btrfs", "subvolume", "snapshot", src, dest)
}
func (a *discoveryAddresser) Register(cancel chan bool, id string, address string, server Server) (retErr error) {
	protolog.Info(&log.StartRegister{id, address})
	defer func() {
		protolog.Info(&log.FinishRegister{id, address, errorToString(retErr)})
	}()
	var once sync.Once
	versionChan := make(chan int64)
	internalCancel := make(chan bool)
	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		if err := a.announceState(id, address, server, versionChan, internalCancel); err != nil {
			once.Do(func() {
				retErr = err
				close(internalCancel)
			})
		}
	}()
	go func() {
		defer wg.Done()
		if err := a.fillRoles(id, server, versionChan, internalCancel); err != nil {
			once.Do(func() {
				retErr = err
				close(internalCancel)
			})
		}
	}()
	go func() {
		defer wg.Done()
		select {
		case <-cancel:
			once.Do(func() {
				retErr = ErrCancelled
				close(internalCancel)
			})
		case <-internalCancel:
		}
	}()
	wg.Wait()
	return
}
Beispiel #10
0
func (d *directory) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
	defer func() {
		protolog.Info(&DirectoryAttr{&d.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
	}()
	if d.Write {
		a.Mode = os.ModeDir | 0775
	} else {
		a.Mode = os.ModeDir | 0555
	}
	return nil
}
Beispiel #11
0
func (f *file) Read(ctx context.Context, request *fuse.ReadRequest, response *fuse.ReadResponse) (retErr error) {
	defer func() {
		protolog.Info(&FileRead{&f.Node, errorToString(retErr)})
	}()
	buffer := bytes.NewBuffer(make([]byte, 0, request.Size))
	if err := pfsutil.GetFile(f.fs.apiClient, f.RepoName, f.CommitID, f.Path, request.Offset, int64(request.Size), buffer); err != nil {
		return err
	}
	response.Data = buffer.Bytes()
	return nil
}
Beispiel #12
0
func (d *directory) Lookup(ctx context.Context, name string) (result fs.Node, retErr error) {
	defer func() {
		protolog.Info(&DirectoryLookup{&d.Node, name, getNode(result), errorToString(retErr)})
	}()
	if d.File.Commit.Repo.Name == "" {
		return d.lookUpRepo(ctx, name)
	}
	if d.File.Commit.Id == "" {
		return d.lookUpCommit(ctx, name)
	}
	return d.lookUpFile(ctx, name)
}
Beispiel #13
0
func execSubvolumeFindNew(commit string, fromCommit string, out io.Writer) (retErr error) {
	defer func() {
		protolog.Info(&SubvolumeFindNew{commit, fromCommit, errorToString(retErr)})
	}()
	if fromCommit == "" {
		return executil.RunStdout(out, "btrfs", "subvolume", "find-new", commit, "0")
	}
	transid, err := execTransID(fromCommit)
	if err != nil {
		return err
	}
	return executil.RunStdout(out, "btrfs", "subvolume", "find-new", commit, transid)
}
Beispiel #14
0
func (d *directory) Mkdir(ctx context.Context, request *fuse.MkdirRequest) (result fs.Node, retErr error) {
	defer func() {
		protolog.Info(&DirectoryMkdir{&d.Node, getNode(result), errorToString(retErr)})
	}()
	if d.File.Commit.Id == "" {
		return nil, fuse.EPERM
	}
	if err := pfsutil.MakeDirectory(d.fs.apiClient, d.File.Commit.Repo.Name, d.File.Commit.Id, path.Join(d.File.Path, request.Name)); err != nil {
		return nil, err
	}
	localResult := d.copy()
	localResult.File.Path = path.Join(localResult.File.Path, request.Name)
	return localResult, nil
}
Beispiel #15
0
func (f *filesystem) Root() (result fs.Node, retErr error) {
	defer func() {
		protolog.Info(&Root{&f.Filesystem, getNode(result), errorToString(retErr)})
	}()
	return &directory{
		f,
		Node{&pfs.File{
			Commit: &pfs.Commit{
				Repo: &pfs.Repo{},
			},
		},
			true},
	}, nil
}
Beispiel #16
0
func (a *discoveryAddresser) GetShardToMasterAddress(version int64) (result map[uint64]string, retErr error) {
	defer func() {
		protolog.Info(&log.GetShardToMasterAddress{version, result, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return nil, err
	}
	_result := make(map[uint64]string)
	for shard, shardAddresses := range addresses.Addresses {
		_result[shard] = shardAddresses.Master
	}
	return _result, nil
}
Beispiel #17
0
func (a *discoveryAddresser) GetReplicaAddresses(shard uint64, version int64) (result map[string]bool, retErr error) {
	defer func() {
		protolog.Info(&log.GetReplicaAddresses{shard, version, result, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return nil, err
	}
	shardAddresses, ok := addresses.Addresses[shard]
	if !ok {
		return nil, fmt.Errorf("shard %d not found", shard)
	}
	return shardAddresses.Replicas, nil
}
Beispiel #18
0
func (a *discoveryAddresser) GetMasterAddress(shard uint64, version int64) (result string, ok bool, retErr error) {
	defer func() {
		protolog.Info(&log.GetMasterAddress{shard, version, result, ok, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return "", false, err
	}
	shardAddresses, ok := addresses.Addresses[shard]
	if !ok {
		return "", false, nil
	}
	return shardAddresses.Master, true, nil
}
Beispiel #19
0
func (f *file) Write(ctx context.Context, request *fuse.WriteRequest, response *fuse.WriteResponse) (retErr error) {
	defer func() {
		protolog.Info(&FileWrite{&f.Node, errorToString(retErr)})
	}()
	written, err := pfsutil.PutFile(f.fs.apiClient, f.File.Commit.Repo.Name, f.File.Commit.Id, f.File.Path, request.Offset, bytes.NewReader(request.Data))
	if err != nil {
		return err
	}
	response.Size = written
	if f.size < request.Offset+int64(written) {
		f.size = request.Offset + int64(written)
	}
	return nil
}
Beispiel #20
0
func (a *apiServer) CreatePipelineRun(ctx context.Context, request *pps.CreatePipelineRunRequest) (*pps.PipelineRun, error) {
	pipelineRun := &pps.PipelineRun{
		Id:         strings.Replace(uuid.NewV4().String(), "-", "", -1),
		PipelineId: request.PipelineId,
	}
	if err := a.storeClient.CreatePipelineRun(pipelineRun); err != nil {
		return nil, err
	}
	protolog.Info(
		&CreatedPipelineRun{
			PipelineRun: pipelineRun,
		},
	)
	return pipelineRun, nil
}
Beispiel #21
0
func (d *directory) Create(ctx context.Context, request *fuse.CreateRequest, response *fuse.CreateResponse) (result fs.Node, _ fs.Handle, retErr error) {
	defer func() {
		protolog.Info(&DirectoryCreate{&d.Node, getNode(result), errorToString(retErr)})
	}()
	if d.File.Commit.Id == "" {
		return nil, 0, fuse.EPERM
	}
	directory := d.copy()
	directory.File.Path = path.Join(directory.File.Path, request.Name)
	localResult := &file{*directory, 0, 0}
	handle, err := localResult.Open(ctx, nil, nil)
	if err != nil {
		return nil, nil, err
	}
	return localResult, handle, nil
}
Beispiel #22
0
func (d *directory) ReadDirAll(ctx context.Context) (result []fuse.Dirent, retErr error) {
	defer func() {
		var dirents []*Dirent
		for _, dirent := range result {
			dirents = append(dirents, &Dirent{dirent.Inode, dirent.Name})
		}
		protolog.Info(&DirectoryReadDirAll{&d.Node, dirents, errorToString(retErr)})
	}()
	if d.File.Commit.Repo.Name == "" {
		return d.readRepos(ctx)
	}
	if d.File.Commit.Id == "" {
		return d.readCommits(ctx)
	}
	return d.readFiles(ctx)
}
Beispiel #23
0
func (a *apiServer) CreatePipelineRun(ctx context.Context, request *pps.CreatePipelineRunRequest) (*pps.PipelineRun, error) {
	pipelineRun := &pps.PipelineRun{
		Id:         strings.Replace(uuid.NewV4().String(), "-", "", -1),
		PipelineId: request.PipelineId,
	}
	// TODO(pedge): should be transactional with call to CreatePipelineRunStatus
	if err := a.storeClient.CreatePipelineRun(pipelineRun); err != nil {
		return nil, err
	}
	if err := a.storeClient.CreatePipelineRunStatus(pipelineRun.Id, pps.PipelineRunStatusType_PIPELINE_RUN_STATUS_TYPE_CREATED); err != nil {
		return nil, err
	}
	protolog.Info(
		&CreatedPipelineRun{
			PipelineRun: pipelineRun,
		},
	)
	return pipelineRun, nil
}
Beispiel #24
0
func (f *file) Attr(ctx context.Context, a *fuse.Attr) (retErr error) {
	defer func() {
		protolog.Info(&FileAttr{&f.Node, &Attr{uint32(a.Mode)}, errorToString(retErr)})
	}()
	fileInfo, err := pfsutil.InspectFile(
		f.fs.apiClient,
		f.RepoName,
		f.CommitID,
		f.Path,
	)
	if err != nil {
		return err
	}
	if fileInfo != nil {
		a.Size = fileInfo.SizeBytes
	}
	a.Mode = 0666
	return nil
}
Beispiel #25
0
func (a *discoveryAddresser) GetShardToReplicaAddresses(version int64) (result map[uint64]map[string]bool, retErr error) {
	defer func() {
		// We need resultPrime is because proto3 can't do maps of maps.
		resultPrime := make(map[uint64]*log.ReplicaAddresses)
		for shard, addresses := range result {
			resultPrime[shard] = &log.ReplicaAddresses{addresses}
		}
		protolog.Info(&log.GetShardToReplicaAddresses{version, resultPrime, errorToString(retErr)})
	}()
	addresses, err := a.getAddresses(version)
	if err != nil {
		return nil, err
	}
	_result := make(map[uint64]map[string]bool)
	for shard, shardAddresses := range addresses.Addresses {
		_result[shard] = shardAddresses.Replicas
	}
	return _result, nil
}
Beispiel #26
0
func (a *discoveryAddresser) Version() (result int64, retErr error) {
	defer func() {
		protolog.Info(&log.Version{result, errorToString(retErr)})
	}()
	minVersion := int64(math.MaxInt64)
	encodedServerStates, err := a.discoveryClient.GetAll(a.serverStateDir())
	if err != nil {
		return 0, err
	}
	for _, encodedServerState := range encodedServerStates {
		serverState, err := decodeServerState(encodedServerState)
		if err != nil {
			return 0, err
		}
		if serverState.Version < minVersion {
			minVersion = serverState.Version
		}
	}
	return minVersion, nil
}
Beispiel #27
0
func (f *file) Read(ctx context.Context, request *fuse.ReadRequest, response *fuse.ReadResponse) (retErr error) {
	defer func() {
		protolog.Info(&FileRead{&f.Node, errorToString(retErr)})
	}()
	var buffer bytes.Buffer
	if err := pfsutil.GetFile(
		f.fs.apiClient,
		f.File.Commit.Repo.Name,
		f.File.Commit.Id,
		f.File.Path,
		request.Offset,
		int64(request.Size),
		f.fs.Shard,
		&buffer,
	); err != nil {
		return err
	}
	response.Data = buffer.Bytes()
	return nil
}
Beispiel #28
0
func execSubvolumeList(path string, fromCommit string, ascending bool, out io.Writer) (retErr error) {
	defer func() {
		protolog.Info(&SubvolumeList{path, fromCommit, ascending, errorToString(retErr)})
	}()
	var sort string
	if ascending {
		sort = "+ogen"
	} else {
		sort = "-ogen"
	}

	if fromCommit == "" {
		return executil.RunStdout(out, "btrfs", "subvolume", "list", "-a", "--sort", sort, path)
	}
	transid, err := execTransID(fromCommit)
	if err != nil {
		return err
	}
	return executil.RunStdout(out, "btrfs", "subvolume", "list", "-aC", "+"+transid, "--sort", sort, path)
}
Beispiel #29
0
func (a *discoveryAddresser) fillRoles(
	id string,
	server Server,
	versionChan chan int64,
	cancel chan bool,
) error {
	oldRoles := make(map[int64]proto.ServerRole)
	return a.discoveryClient.WatchAll(
		a.serverRoleKey(id),
		cancel,
		func(encodedServerRoles map[string]string) error {
			roles := make(map[int64]proto.ServerRole)
			var versions int64Slice
			// Decode the roles
			for _, encodedServerRole := range encodedServerRoles {
				var serverRole proto.ServerRole
				if err := jsonpb.UnmarshalString(encodedServerRole, &serverRole); err != nil {
					return err
				}
				roles[serverRole.Version] = serverRole
				versions = append(versions, serverRole.Version)
			}
			sort.Sort(versions)
			if len(versions) > 2 {
				versions = versions[0:2]
			}
			// For each new version bring the server up to date
			for _, version := range versions {
				if _, ok := oldRoles[version]; ok {
					// we've already seen these roles, so nothing to do here
					continue
				}
				serverRole := roles[version]
				var wg sync.WaitGroup
				var addShardErr error
				for _, shard := range shards(serverRole) {
					if !containsShard(oldRoles, shard) {
						wg.Add(1)
						shard := shard
						go func() {
							defer wg.Done()
							if err := server.AddShard(shard); err != nil && addShardErr == nil {
								addShardErr = err
							}
						}()
					}
				}
				wg.Wait()
				if addShardErr != nil {
					protolog.Info(&log.AddServerRole{&serverRole, addShardErr.Error()})
					return addShardErr
				}
				protolog.Info(&log.AddServerRole{&serverRole, ""})
				oldRoles[version] = serverRole
				versionChan <- version
			}
			// See if there are any old roles that aren't needed
			for version, serverRole := range oldRoles {
				var wg sync.WaitGroup
				var removeShardErr error
				if _, ok := roles[version]; ok {
					// these roles haven't expired yet, so nothing to do
					continue
				}
				for _, shard := range shards(serverRole) {
					if !containsShard(roles, shard) {
						wg.Add(1)
						shard := shard
						go func(shard uint64) {
							defer wg.Done()
							if err := server.RemoveShard(shard); err != nil && removeShardErr == nil {
								removeShardErr = err
							}
						}(shard)
					}
				}
				wg.Wait()
				if removeShardErr != nil {
					protolog.Info(&log.RemoveServerRole{&serverRole, removeShardErr.Error()})
					return removeShardErr
				}
				protolog.Info(&log.RemoveServerRole{&serverRole, ""})
			}
			oldRoles = make(map[int64]proto.ServerRole)
			for _, version := range versions {
				oldRoles[version] = roles[version]
			}
			return nil
		},
	)
}
Beispiel #30
0
func (a *discoveryAddresser) AssignRoles(cancel chan bool) (retErr error) {
	protolog.Info(&log.StartAssignRoles{})
	defer func() {
		protolog.Info(&log.FinishAssignRoles{errorToString(retErr)})
	}()
	var version int64
	oldServers := make(map[string]bool)
	oldRoles := make(map[string]*proto.ServerRole)
	oldMasters := make(map[uint64]string)
	oldReplicas := make(map[uint64][]string)
	var oldMinVersion int64
	// Reconstruct state from a previous run
	serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
	if err != nil {
		return err
	}
	for _, encodedServerRole := range serverRoles {
		serverRole, err := decodeServerRole(encodedServerRole)
		if err != nil {
			return err
		}
		if oldServerRole, ok := oldRoles[serverRole.Id]; !ok || oldServerRole.Version < serverRole.Version {
			oldRoles[serverRole.Id] = serverRole
			oldServers[serverRole.Id] = true
		}
		if version < serverRole.Version+1 {
			version = serverRole.Version + 1
		}
	}
	for _, oldServerRole := range oldRoles {
		for shard := range oldServerRole.Masters {
			oldMasters[shard] = oldServerRole.Id
		}
		for shard := range oldServerRole.Replicas {
			oldReplicas[shard] = append(oldReplicas[shard], oldServerRole.Id)
		}
	}
	err = a.discoveryClient.WatchAll(a.serverStateDir(), cancel,
		func(encodedServerStates map[string]string) error {
			if len(encodedServerStates) == 0 {
				return nil
			}
			newServerStates := make(map[string]*proto.ServerState)
			shardLocations := make(map[uint64][]string)
			newRoles := make(map[string]*proto.ServerRole)
			newMasters := make(map[uint64]string)
			newReplicas := make(map[uint64][]string)
			masterRolesPerServer := a.sharder.NumShards() / uint64(len(encodedServerStates))
			masterRolesRemainder := a.sharder.NumShards() % uint64(len(encodedServerStates))
			replicaRolesPerServer := (a.sharder.NumShards() * (a.sharder.NumReplicas())) / uint64(len(encodedServerStates))
			replicaRolesRemainder := (a.sharder.NumShards() * (a.sharder.NumReplicas())) % uint64(len(encodedServerStates))
			for _, encodedServerState := range encodedServerStates {
				serverState, err := decodeServerState(encodedServerState)
				if err != nil {
					return err
				}
				newServerStates[serverState.Id] = serverState
				newRoles[serverState.Id] = &proto.ServerRole{
					Id:       serverState.Id,
					Version:  version,
					Masters:  make(map[uint64]bool),
					Replicas: make(map[uint64]bool),
				}
				for shard := range serverState.Shards {
					shardLocations[shard] = append(shardLocations[shard], serverState.Id)
				}
			}
			// See if there's any roles we can delete
			minVersion := int64(math.MaxInt64)
			for _, serverState := range newServerStates {
				if serverState.Version < minVersion {
					minVersion = serverState.Version
				}
			}
			// Delete roles that no servers are using anymore
			if minVersion > oldMinVersion {
				oldMinVersion = minVersion
				serverRoles, err := a.discoveryClient.GetAll(a.serverRoleDir())
				if err != nil {
					return err
				}
				for key, encodedServerRole := range serverRoles {
					serverRole, err := decodeServerRole(encodedServerRole)
					if err != nil {
						return err
					}
					if serverRole.Version < minVersion {
						if err := a.discoveryClient.Delete(key); err != nil {
							return err
						}
						protolog.Info(&log.DeleteServerRole{serverRole})
					}
				}
			}
			// if the servers are identical to last time then we know we'll
			// assign shards the same way
			if sameServers(oldServers, newServerStates) {
				return nil
			}
		Master:
			for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
				if id, ok := oldMasters[shard]; ok {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				for _, id := range oldReplicas[shard] {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				for _, id := range shardLocations[shard] {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				for id := range newServerStates {
					if assignMaster(newRoles, newMasters, id, shard, masterRolesPerServer, &masterRolesRemainder) {
						continue Master
					}
				}
				protolog.Error(&log.FailedToAssignRoles{
					ServerStates: newServerStates,
					NumShards:    a.sharder.NumShards(),
					NumReplicas:  a.sharder.NumReplicas(),
				})
				return nil
			}
			for replica := uint64(0); replica < a.sharder.NumReplicas(); replica++ {
			Replica:
				for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
					if id, ok := oldMasters[shard]; ok {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for _, id := range oldReplicas[shard] {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for _, id := range shardLocations[shard] {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for id := range newServerStates {
						if assignReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer, &replicaRolesRemainder) {
							continue Replica
						}
					}
					for id := range newServerStates {
						if swapReplica(newRoles, newMasters, newReplicas, id, shard, replicaRolesPerServer) {
							continue Replica
						}
					}
					protolog.Error(&log.FailedToAssignRoles{
						ServerStates: newServerStates,
						NumShards:    a.sharder.NumShards(),
						NumReplicas:  a.sharder.NumReplicas(),
					})
					return nil
				}
			}
			addresses := proto.Addresses{
				Version:   version,
				Addresses: make(map[uint64]*proto.ShardAddresses),
			}
			for shard := uint64(0); shard < a.sharder.NumShards(); shard++ {
				addresses.Addresses[shard] = &proto.ShardAddresses{Replicas: make(map[string]bool)}
			}
			for id, serverRole := range newRoles {
				encodedServerRole, err := marshaler.MarshalToString(serverRole)
				if err != nil {
					return err
				}
				if err := a.discoveryClient.Set(a.serverRoleKeyVersion(id, version), encodedServerRole, 0); err != nil {
					return err
				}
				protolog.Info(&log.SetServerRole{serverRole})
				address := newServerStates[id].Address
				for shard := range serverRole.Masters {
					shardAddresses := addresses.Addresses[shard]
					shardAddresses.Master = address
					addresses.Addresses[shard] = shardAddresses
				}
				for shard := range serverRole.Replicas {
					shardAddresses := addresses.Addresses[shard]
					shardAddresses.Replicas[address] = true
					addresses.Addresses[shard] = shardAddresses
				}
			}
			encodedAddresses, err := marshaler.MarshalToString(&addresses)
			if err != nil {
				return err
			}
			if err := a.discoveryClient.Set(a.addressesKey(version), encodedAddresses, 0); err != nil {
				return err
			}
			protolog.Info(&log.SetAddresses{&addresses})
			version++
			oldServers = make(map[string]bool)
			for id := range newServerStates {
				oldServers[id] = true
			}
			oldRoles = newRoles
			oldMasters = newMasters
			oldReplicas = newReplicas
			return nil
		})
	if err == discovery.ErrCancelled {
		return ErrCancelled
	}
	return err
}