// Copy blocks from src to dst; only those that are missing in dst are copied func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, blobSigningKey string) error { total := len(toBeCopied) startedAt := time.Now() for done, locator := range toBeCopied { if done == 0 { log.Printf("Copying data block %d of %d (%.2f%% done): %v", done+1, total, float64(done)/float64(total)*100, locator) } else { timePerBlock := time.Since(startedAt) / time.Duration(done) log.Printf("Copying data block %d of %d (%.2f%% done, %v est. time remaining): %v", done+1, total, float64(done)/float64(total)*100, timePerBlock*time.Duration(total-done), locator) } getLocator := locator expiresAt := time.Now().AddDate(0, 0, 1) if blobSigningKey != "" { getLocator = keepclient.SignLocator(getLocator, kcSrc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey)) } reader, len, _, err := kcSrc.Get(getLocator) if err != nil { return fmt.Errorf("Error getting block: %v %v", locator, err) } _, _, err = kcDst.PutHR(getLocator[:32], reader, len) if err != nil { return fmt.Errorf("Error copying data block: %v %v", locator, err) } } log.Printf("Successfully copied to destination %d blocks.", total) return nil }
// Get block headers from keep. Log any errors. func performKeepBlockCheck(kc *keepclient.KeepClient, blobSignatureTTL time.Duration, blobSigningKey string, blockLocators []string, verbose bool) error { totalBlocks := len(blockLocators) notFoundBlocks := 0 current := 0 for _, locator := range blockLocators { current++ if verbose { log.Printf("Verifying block %d of %d: %v", current, totalBlocks, locator) } getLocator := locator if blobSigningKey != "" { expiresAt := time.Now().AddDate(0, 0, 1) getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, blobSignatureTTL, []byte(blobSigningKey)) } _, _, err := kc.Ask(getLocator) if err != nil { notFoundBlocks++ log.Printf("Error verifying block %v: %v", locator, err) } } log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks) if notFoundBlocks > 0 { return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks) } return nil }
// Do a Put in the first and Get from the second, // which should raise block not found error. func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) { // Put a block using kc1 locator, _, err := kc1.PutB([]byte(testData)) c.Assert(err, Equals, nil) locator = strings.Split(locator, "+")[0] _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), blobSignatureTTL, []byte(blobSigningKey))) c.Assert(err, NotNil) c.Check(err.Error(), Equals, "Block not found") }
// SignLocator takes a blobLocator, an apiToken and an expiry time, and // returns a signed locator string. func SignLocator(blobLocator, apiToken string, expiry time.Time) string { return keepclient.SignLocator(blobLocator, apiToken, expiry, blobSignatureTTL, PermissionSecret) }