// PutBlockBlob uploads given stream into a block blob by splitting
// data stream into chunks and uploading as blocks. Commits the block
// list at the end. This is a helper method built on top of PutBlock
// and PutBlockList methods with sequential block ID counting logic.
func putBlockBlob(b storage.BlobStorageClient, container, name string, blob io.Reader, chunkSize int) error {
if chunkSize <= 0 || chunkSize > storage.MaxBlobBlockSize {
chunkSize = storage.MaxBlobBlockSize
}
chunk := make([]byte, chunkSize)
n, err := blob.Read(chunk)
if err != nil && err != io.EOF {
return err
}
blockList := []storage.Block{}
for blockNum := 0; ; blockNum++ {
id := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%011d", blockNum)))
data := chunk[:n]
err = b.PutBlock(container, name, id, data)
if err != nil {
return err
}
blockList = append(blockList, storage.Block{id, storage.BlockStatusLatest})
// Read next block
n, err = blob.Read(chunk)
if err != nil && err != io.EOF {
return err
}
if err == io.EOF {
break
}
}
return b.PutBlockList(container, name, blockList)
}
func resourceArmStorageBlobBlockUploadFromSource(container, name, source string, client *storage.BlobStorageClient, parallelism, attempts int) error {
workerCount := parallelism * runtime.NumCPU()
file, err := os.Open(source)
if err != nil {
return fmt.Errorf("Error opening source file for upload %q: %s", source, err)
}
defer file.Close()
blockList, parts, err := resourceArmStorageBlobBlockSplit(file)
if err != nil {
return fmt.Errorf("Error reading and splitting source file for upload %q: %s", source, err)
}
wg := &sync.WaitGroup{}
blocks := make(chan resourceArmStorageBlobBlock, len(parts))
errors := make(chan error, len(parts))
wg.Add(len(parts))
for _, p := range parts {
blocks <- p
}
close(blocks)
for i := 0; i < workerCount; i++ {
go resourceArmStorageBlobBlockUploadWorker(resourceArmStorageBlobBlockUploadContext{
client: client,
source: source,
container: container,
name: name,
blocks: blocks,
errors: errors,
wg: wg,
attempts: attempts,
})
}
wg.Wait()
if len(errors) > 0 {
return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors)
}
err = client.PutBlockList(container, name, blockList)
if err != nil {
return fmt.Errorf("Error updating block list for source file %q: %s", source, err)
}
return nil
}