func backupFiles(mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fes []FileEntry, replicationPosition replication.Position, backupConcurrency int) (err error) { sema := sync2.NewSemaphore(backupConcurrency, 0) rec := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for i, fe := range fes { wg.Add(1) go func(i int, fe FileEntry) { defer wg.Done() // wait until we are ready to go, skip if we already // encountered an error sema.Acquire() defer sema.Release() if rec.HasErrors() { return } // open the source file for reading source, err := fe.open(mysqld.Cnf(), true) if err != nil { rec.RecordError(err) return } defer source.Close() // open the destination file for writing, and a buffer name := fmt.Sprintf("%v", i) wc, err := bh.AddFile(name) if err != nil { rec.RecordError(fmt.Errorf("cannot add file: %v", err)) return } defer func() { rec.RecordError(wc.Close()) }() dst := bufio.NewWriterSize(wc, 2*1024*1024) // create the hasher and the tee on top hasher := newHasher() tee := io.MultiWriter(dst, hasher) // create the gzip compression filter gzip, err := cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED) if err != nil { rec.RecordError(fmt.Errorf("cannot create gziper: %v", err)) return } // copy from the source file to gzip to tee to output file and hasher _, err = io.Copy(gzip, source) if err != nil { rec.RecordError(fmt.Errorf("cannot copy data: %v", err)) return } // close gzip to flush it, after that the hash is good if err = gzip.Close(); err != nil { rec.RecordError(fmt.Errorf("cannot close gzip: %v", err)) return } // flush the buffer to finish writing, save the hash rec.RecordError(dst.Flush()) fes[i].Hash = hasher.HashString() }(i, fe) } wg.Wait() if rec.HasErrors() { return rec.Error() } // open the MANIFEST wc, err := bh.AddFile(backupManifest) if err != nil { return fmt.Errorf("cannot add %v to backup: %v", backupManifest, err) } defer func() { if closeErr := wc.Close(); err == nil { err = closeErr } }() // JSON-encode and write the MANIFEST bm := &BackupManifest{ FileEntries: fes, Position: replicationPosition, } data, err := json.MarshalIndent(bm, "", " ") if err != nil { return fmt.Errorf("cannot JSON encode %v: %v", backupManifest, err) } if _, err := wc.Write([]byte(data)); err != nil { return fmt.Errorf("cannot write %v: %v", backupManifest, err) } return nil }