func backupFiles(mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fes []FileEntry, replicationPosition replication.Position, backupConcurrency int) (err error) {
sema := sync2.NewSemaphore(backupConcurrency, 0)
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for i, fe := range fes {
wg.Add(1)
go func(i int, fe FileEntry) {
defer wg.Done()
// wait until we are ready to go, skip if we already
// encountered an error
sema.Acquire()
defer sema.Release()
if rec.HasErrors() {
return
}
// open the source file for reading
source, err := fe.open(mysqld.Cnf(), true)
if err != nil {
rec.RecordError(err)
return
}
defer source.Close()
// open the destination file for writing, and a buffer
name := fmt.Sprintf("%v", i)
wc, err := bh.AddFile(name)
if err != nil {
rec.RecordError(fmt.Errorf("cannot add file: %v", err))
return
}
defer func() { rec.RecordError(wc.Close()) }()
dst := bufio.NewWriterSize(wc, 2*1024*1024)
// create the hasher and the tee on top
hasher := newHasher()
tee := io.MultiWriter(dst, hasher)
// create the gzip compression filter
gzip, err := cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED)
if err != nil {
rec.RecordError(fmt.Errorf("cannot create gziper: %v", err))
return
}
// copy from the source file to gzip to tee to output file and hasher
_, err = io.Copy(gzip, source)
if err != nil {
rec.RecordError(fmt.Errorf("cannot copy data: %v", err))
return
}
// close gzip to flush it, after that the hash is good
if err = gzip.Close(); err != nil {
rec.RecordError(fmt.Errorf("cannot close gzip: %v", err))
return
}
// flush the buffer to finish writing, save the hash
rec.RecordError(dst.Flush())
fes[i].Hash = hasher.HashString()
}(i, fe)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// open the MANIFEST
wc, err := bh.AddFile(backupManifest)
if err != nil {
return fmt.Errorf("cannot add %v to backup: %v", backupManifest, err)
}
defer func() {
if closeErr := wc.Close(); err == nil {
err = closeErr
}
}()
// JSON-encode and write the MANIFEST
bm := &BackupManifest{
FileEntries: fes,
Position: replicationPosition,
}
data, err := json.MarshalIndent(bm, "", " ")
if err != nil {
return fmt.Errorf("cannot JSON encode %v: %v", backupManifest, err)
}
if _, err := wc.Write([]byte(data)); err != nil {
return fmt.Errorf("cannot write %v: %v", backupManifest, err)
}
return nil
}
