// ComputePullServers creates a map from block locator to PullServers // with one entry for each under-replicated block. // // This method ignores zero-replica blocks since there are no servers // to pull them from, so callers should feel free to omit them, but // this function will ignore them if they are provided. func ComputePullServers(kc *keepclient.KeepClient, keepServerInfo *keep.ReadServers, blockToDesiredReplication map[blockdigest.DigestWithSize]int, underReplicated BlockSet) (m map[Locator]PullServers) { m = map[Locator]PullServers{} // We use CanonicalString to avoid filling memory with dupicate // copies of the same string. var cs CanonicalString // Servers that are writeable writableServers := map[string]struct{}{} for _, url := range kc.WritableLocalRoots() { writableServers[cs.Get(url)] = struct{}{} } for block := range underReplicated { serversStoringBlock := keepServerInfo.BlockToServers[block] numCopies := len(serversStoringBlock) numCopiesMissing := blockToDesiredReplication[block] - numCopies if numCopiesMissing > 0 { // We expect this to always be true, since the block was listed // in underReplicated. if numCopies > 0 { // Not much we can do with blocks with no copies. // A server's host-port string appears as a key in this map // iff it contains the block. serverHasBlock := map[string]struct{}{} for _, info := range serversStoringBlock { sa := keepServerInfo.KeepServerIndexToAddress[info.ServerIndex] serverHasBlock[cs.Get(sa.URL())] = struct{}{} } roots := keepclient.NewRootSorter(kc.LocalRoots(), block.String()).GetSortedRoots() l := Locator(block) m[l] = CreatePullServers(cs, serverHasBlock, writableServers, roots, numCopiesMissing) } } } return m }
// BuildTrashLists builds list of blocks to be sent to trash queue func BuildTrashLists(kc *keepclient.KeepClient, keepServerInfo *keep.ReadServers, keepBlocksNotInCollections BlockSet) (m map[string]keep.TrashList, err error) { // Servers that are writeable writableServers := map[string]struct{}{} for _, url := range kc.WritableLocalRoots() { writableServers[url] = struct{}{} } _ttl, err := kc.Arvados.Discovery("blobSignatureTtl") if err != nil { return nil, errors.New(fmt.Sprintf("Failed to get blobSignatureTtl, can't build trash lists: %v", err)) } ttl := int64(_ttl.(float64)) // expire unreferenced blocks more than "ttl" seconds old. expiry := time.Now().UTC().Unix() - ttl return buildTrashListsInternal(writableServers, keepServerInfo, expiry, keepBlocksNotInCollections), nil }