// Copy blocks from src to dst; only those that are missing in dst are copied
func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, blobSigningKey string) error {
total := len(toBeCopied)
startedAt := time.Now()
for done, locator := range toBeCopied {
if done == 0 {
log.Printf("Copying data block %d of %d (%.2f%% done): %v", done+1, total,
float64(done)/float64(total)*100, locator)
} else {
timePerBlock := time.Since(startedAt) / time.Duration(done)
log.Printf("Copying data block %d of %d (%.2f%% done, %v est. time remaining): %v", done+1, total,
float64(done)/float64(total)*100, timePerBlock*time.Duration(total-done), locator)
}
getLocator := locator
expiresAt := time.Now().AddDate(0, 0, 1)
if blobSigningKey != "" {
getLocator = keepclient.SignLocator(getLocator, kcSrc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey))
}
reader, len, _, err := kcSrc.Get(getLocator)
if err != nil {
return fmt.Errorf("Error getting block: %v %v", locator, err)
}
_, _, err = kcDst.PutHR(getLocator[:32], reader, len)
if err != nil {
return fmt.Errorf("Error copying data block: %v %v", locator, err)
}
}
log.Printf("Successfully copied to destination %d blocks.", total)
return nil
}
// Do a Put in the first and Get from the second,
// which should raise block not found error.
func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) {
// Put a block using kc1
locator, _, err := kc1.PutB([]byte(testData))
c.Assert(err, Equals, nil)
locator = strings.Split(locator, "+")[0]
_, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), blobSignatureTTL, []byte(blobSigningKey)))
c.Assert(err, NotNil)
c.Check(err.Error(), Equals, "Block not found")
}
func doReads(kc *keepclient.KeepClient, nextLocator chan string) {
for locator := range nextLocator {
rdr, size, url, err := kc.Get(locator)
if err != nil {
log.Print(err)
errorsChan <- struct{}{}
continue
}
n, err := io.Copy(ioutil.Discard, rdr)
rdr.Close()
if n != size || err != nil {
log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
errorsChan <- struct{}{}
continue
// Note we don't count the bytes received in
// partial/corrupt responses: we are measuring
// throughput, not resource consumption.
}
bytesInChan <- uint64(n)
}
}