// ComputePullServers creates a map from block locator to PullServers
// with one entry for each under-replicated block.
//
// This method ignores zero-replica blocks since there are no servers
// to pull them from, so callers should feel free to omit them, but
// this function will ignore them if they are provided.
func ComputePullServers(kc *keepclient.KeepClient,
keepServerInfo *keep.ReadServers,
blockToDesiredReplication map[blockdigest.DigestWithSize]int,
underReplicated BlockSet) (m map[Locator]PullServers) {
m = map[Locator]PullServers{}
// We use CanonicalString to avoid filling memory with dupicate
// copies of the same string.
var cs CanonicalString
// Servers that are writeable
writableServers := map[string]struct{}{}
for _, url := range kc.WritableLocalRoots() {
writableServers[cs.Get(url)] = struct{}{}
}
for block := range underReplicated {
serversStoringBlock := keepServerInfo.BlockToServers[block]
numCopies := len(serversStoringBlock)
numCopiesMissing := blockToDesiredReplication[block] - numCopies
if numCopiesMissing > 0 {
// We expect this to always be true, since the block was listed
// in underReplicated.
if numCopies > 0 {
// Not much we can do with blocks with no copies.
// A server's host-port string appears as a key in this map
// iff it contains the block.
serverHasBlock := map[string]struct{}{}
for _, info := range serversStoringBlock {
sa := keepServerInfo.KeepServerIndexToAddress[info.ServerIndex]
serverHasBlock[cs.Get(sa.URL())] = struct{}{}
}
roots := keepclient.NewRootSorter(kc.LocalRoots(),
block.String()).GetSortedRoots()
l := Locator(block)
m[l] = CreatePullServers(cs, serverHasBlock, writableServers,
roots, numCopiesMissing)
}
}
}
return m
}
// balanceBlock compares current state to desired state for a single
// block, and makes the appropriate ChangeSet calls.
func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) {
debugf("balanceBlock: %v %+v", blkid, blk)
uuids := keepclient.NewRootSorter(bal.serviceRoots, string(blkid[:32])).GetSortedRoots()
hasRepl := make(map[string]Replica, len(bal.serviceRoots))
for _, repl := range blk.Replicas {
hasRepl[repl.UUID] = repl
// TODO: when multiple copies are on one server, use
// the oldest one that doesn't have a timestamp
// collision with other replicas.
}
// number of replicas already found in positions better than
// the position we're contemplating now.
reportedBestRepl := 0
// To be safe we assume two replicas with the same Mtime are
// in fact the same replica being reported more than
// once. len(uniqueBestRepl) is the number of distinct
// replicas in the best rendezvous positions we've considered
// so far.
uniqueBestRepl := make(map[int64]bool, len(bal.serviceRoots))
// pulls is the number of Pull changes we have already
// requested. (For purposes of deciding whether to Pull to
// rendezvous position N, we should assume all pulls we have
// requested on rendezvous positions M<N will be successful.)
pulls := 0
var changes []string
for _, uuid := range uuids {
change := changeNone
srv := bal.KeepServices[uuid]
// TODO: request a Touch if Mtime is duplicated.
repl, ok := hasRepl[srv.UUID]
if ok {
// This service has a replica. We should
// delete it if [1] we already have enough
// distinct replicas in better rendezvous
// positions and [2] this replica's Mtime is
// distinct from all of the better replicas'
// Mtimes.
if !srv.ReadOnly &&
repl.Mtime < bal.MinMtime &&
len(uniqueBestRepl) >= blk.Desired &&
!uniqueBestRepl[repl.Mtime] {
srv.AddTrash(Trash{
SizedDigest: blkid,
Mtime: repl.Mtime,
})
change = changeTrash
} else {
change = changeStay
}
uniqueBestRepl[repl.Mtime] = true
reportedBestRepl++
} else if pulls+reportedBestRepl < blk.Desired &&
len(blk.Replicas) > 0 &&
!srv.ReadOnly {
// This service doesn't have a replica. We
// should pull one to this server if we don't
// already have enough (existing+requested)
// replicas in better rendezvous positions.
srv.AddPull(Pull{
SizedDigest: blkid,
Source: blk.Replicas[0].KeepService,
})
pulls++
change = changePull
}
if bal.Dumper != nil {
changes = append(changes, fmt.Sprintf("%s:%d=%s,%d", srv.ServiceHost, srv.ServicePort, changeName[change], repl.Mtime))
}
}
if bal.Dumper != nil {
bal.Dumper.Printf("%s have=%d want=%d %s", blkid, len(blk.Replicas), blk.Desired, strings.Join(changes, " "))
}
}
