// PutBlockBlob uploads given stream into a block blob by splitting // data stream into chunks and uploading as blocks. Commits the block // list at the end. This is a helper method built on top of PutBlock // and PutBlockList methods with sequential block ID counting logic. func putBlockBlob(b storage.BlobStorageClient, container, name string, blob io.Reader, chunkSize int) error { if chunkSize <= 0 || chunkSize > storage.MaxBlobBlockSize { chunkSize = storage.MaxBlobBlockSize } chunk := make([]byte, chunkSize) n, err := blob.Read(chunk) if err != nil && err != io.EOF { return err } blockList := []storage.Block{} for blockNum := 0; ; blockNum++ { id := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%011d", blockNum))) data := chunk[:n] err = b.PutBlock(container, name, id, data) if err != nil { return err } blockList = append(blockList, storage.Block{id, storage.BlockStatusLatest}) // Read next block n, err = blob.Read(chunk) if err != nil && err != io.EOF { return err } if err == io.EOF { break } } return b.PutBlockList(container, name, blockList) }
func resourceArmStorageBlobBlockUploadFromSource(container, name, source string, client *storage.BlobStorageClient, parallelism, attempts int) error { workerCount := parallelism * runtime.NumCPU() file, err := os.Open(source) if err != nil { return fmt.Errorf("Error opening source file for upload %q: %s", source, err) } defer file.Close() blockList, parts, err := resourceArmStorageBlobBlockSplit(file) if err != nil { return fmt.Errorf("Error reading and splitting source file for upload %q: %s", source, err) } wg := &sync.WaitGroup{} blocks := make(chan resourceArmStorageBlobBlock, len(parts)) errors := make(chan error, len(parts)) wg.Add(len(parts)) for _, p := range parts { blocks <- p } close(blocks) for i := 0; i < workerCount; i++ { go resourceArmStorageBlobBlockUploadWorker(resourceArmStorageBlobBlockUploadContext{ client: client, source: source, container: container, name: name, blocks: blocks, errors: errors, wg: wg, attempts: attempts, }) } wg.Wait() if len(errors) > 0 { return fmt.Errorf("Error while uploading source file %q: %s", source, <-errors) } err = client.PutBlockList(container, name, blockList) if err != nil { return fmt.Errorf("Error updating block list for source file %q: %s", source, err) } return nil }