// ComputePullServers creates a map from block locator to PullServers
// with one entry for each under-replicated block.
//
// This method ignores zero-replica blocks since there are no servers
// to pull them from, so callers should feel free to omit them, but
// this function will ignore them if they are provided.
func ComputePullServers(kc *keepclient.KeepClient,
keepServerInfo *keep.ReadServers,
blockToDesiredReplication map[blockdigest.DigestWithSize]int,
underReplicated BlockSet) (m map[Locator]PullServers) {
m = map[Locator]PullServers{}
// We use CanonicalString to avoid filling memory with dupicate
// copies of the same string.
var cs CanonicalString
// Servers that are writeable
writableServers := map[string]struct{}{}
for _, url := range kc.WritableLocalRoots() {
writableServers[cs.Get(url)] = struct{}{}
}
for block := range underReplicated {
serversStoringBlock := keepServerInfo.BlockToServers[block]
numCopies := len(serversStoringBlock)
numCopiesMissing := blockToDesiredReplication[block] - numCopies
if numCopiesMissing > 0 {
// We expect this to always be true, since the block was listed
// in underReplicated.
if numCopies > 0 {
// Not much we can do with blocks with no copies.
// A server's host-port string appears as a key in this map
// iff it contains the block.
serverHasBlock := map[string]struct{}{}
for _, info := range serversStoringBlock {
sa := keepServerInfo.KeepServerIndexToAddress[info.ServerIndex]
serverHasBlock[cs.Get(sa.URL())] = struct{}{}
}
roots := keepclient.NewRootSorter(kc.LocalRoots(),
block.String()).GetSortedRoots()
l := Locator(block)
m[l] = CreatePullServers(cs, serverHasBlock, writableServers,
roots, numCopiesMissing)
}
}
}
return m
}
// BuildTrashLists builds list of blocks to be sent to trash queue
func BuildTrashLists(kc *keepclient.KeepClient,
keepServerInfo *keep.ReadServers,
keepBlocksNotInCollections BlockSet) (m map[string]keep.TrashList, err error) {
// Servers that are writeable
writableServers := map[string]struct{}{}
for _, url := range kc.WritableLocalRoots() {
writableServers[url] = struct{}{}
}
_ttl, err := kc.Arvados.Discovery("blobSignatureTtl")
if err != nil {
return nil, errors.New(fmt.Sprintf("Failed to get blobSignatureTtl, can't build trash lists: %v", err))
}
ttl := int64(_ttl.(float64))
// expire unreferenced blocks more than "ttl" seconds old.
expiry := time.Now().UTC().Unix() - ttl
return buildTrashListsInternal(writableServers, keepServerInfo, expiry, keepBlocksNotInCollections), nil
}