func (nhw *namedHasherWriter) Open() (err error) {
// The pipeline looks like this:
//
// +---> buffer +---> file
// | 32K
// buffer +---> gzip +---> tee +
// 32K |
// +---> hasher
//
// The buffer in front of gzip is needed so that the data is
// compressed only when there's a reasonable amount of it.
filename := fmt.Sprintf(nhw.filenamePattern, nhw.currentIndex)
nhw.file, err = os.Create(filename)
if err != nil {
return
}
nhw.fileBuffer = bufio.NewWriterSize(nhw.file, 32*1024)
nhw.hasher = newHasher()
tee := io.MultiWriter(nhw.fileBuffer, nhw.hasher)
// create the gzip compression filter
nhw.gzip, err = cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED)
if err != nil {
return
}
nhw.inputBuffer = bufio2.NewAsyncWriterSize(nhw.gzip, 32*1024, 3)
return
}
// custom function to serve files
func sendFile(rw http.ResponseWriter, req *http.Request, path string) {
log.Infof("serve %v %v", req.URL.Path, path)
file, err := os.Open(path)
if err != nil {
http.NotFound(rw, req)
return
}
defer file.Close()
fileinfo, err := file.Stat()
if err != nil {
http.NotFound(rw, req)
return
}
// for directories, or for files smaller than 1k, use library
if fileinfo.Mode().IsDir() || fileinfo.Size() < 1024 {
http.ServeFile(rw, req, path)
return
}
// supports If-Modified-Since header
if t, err := time.Parse(http.TimeFormat, req.Header.Get("If-Modified-Since")); err == nil && fileinfo.ModTime().Before(t.Add(1*time.Second)) {
rw.WriteHeader(http.StatusNotModified)
return
}
// support Accept-Encoding header
var writer io.Writer = rw
var reader io.Reader = file
if !strings.HasSuffix(path, ".gz") {
ae := req.Header.Get("Accept-Encoding")
if strings.Contains(ae, "gzip") {
gz, err := cgzip.NewWriterLevel(rw, cgzip.Z_BEST_SPEED)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
rw.Header().Set("Content-Encoding", "gzip")
defer gz.Close()
writer = gz
}
}
// add content-length if we know it
if writer == rw && reader == file {
rw.Header().Set("Content-Length", fmt.Sprintf("%v", fileinfo.Size()))
}
// and just copy content out
rw.Header().Set("Last-Modified", fileinfo.ModTime().UTC().Format(http.TimeFormat))
rw.WriteHeader(http.StatusOK)
if _, err := io.Copy(writer, reader); err != nil {
log.Warningf("transfer failed %v: %v", path, err)
}
}
// NewGzipHandler is an HTTP handler used to compress responses if supported by the client
func NewGzipHandler(fn http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !GetConfig().Gzip || !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
fn(w, r)
return
}
w.Header().Set("Content-Encoding", "gzip")
gz, _ := cgzip.NewWriterLevel(w, cgzip.Z_BEST_SPEED)
defer gz.Close()
fn(&gzipResponseWriter{Writer: gz, ResponseWriter: w}, r)
}
}
func backupFiles(mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fes []FileEntry, replicationPosition replication.Position, backupConcurrency int) (err error) {
sema := sync2.NewSemaphore(backupConcurrency, 0)
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for i, fe := range fes {
wg.Add(1)
go func(i int, fe FileEntry) {
defer wg.Done()
// wait until we are ready to go, skip if we already
// encountered an error
sema.Acquire()
defer sema.Release()
if rec.HasErrors() {
return
}
// open the source file for reading
source, err := fe.open(mysqld.Cnf(), true)
if err != nil {
rec.RecordError(err)
return
}
defer source.Close()
// open the destination file for writing, and a buffer
name := fmt.Sprintf("%v", i)
wc, err := bh.AddFile(name)
if err != nil {
rec.RecordError(fmt.Errorf("cannot add file: %v", err))
return
}
defer func() { rec.RecordError(wc.Close()) }()
dst := bufio.NewWriterSize(wc, 2*1024*1024)
// create the hasher and the tee on top
hasher := newHasher()
tee := io.MultiWriter(dst, hasher)
// create the gzip compression filter
gzip, err := cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED)
if err != nil {
rec.RecordError(fmt.Errorf("cannot create gziper: %v", err))
return
}
// copy from the source file to gzip to tee to output file and hasher
_, err = io.Copy(gzip, source)
if err != nil {
rec.RecordError(fmt.Errorf("cannot copy data: %v", err))
return
}
// close gzip to flush it, after that the hash is good
if err = gzip.Close(); err != nil {
rec.RecordError(fmt.Errorf("cannot close gzip: %v", err))
return
}
// flush the buffer to finish writing, save the hash
rec.RecordError(dst.Flush())
fes[i].Hash = hasher.HashString()
}(i, fe)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// open the MANIFEST
wc, err := bh.AddFile(backupManifest)
if err != nil {
return fmt.Errorf("cannot add %v to backup: %v", backupManifest, err)
}
defer func() {
if closeErr := wc.Close(); err == nil {
err = closeErr
}
}()
// JSON-encode and write the MANIFEST
bm := &BackupManifest{
FileEntries: fes,
Position: replicationPosition,
}
data, err := json.MarshalIndent(bm, "", " ")
if err != nil {
return fmt.Errorf("cannot JSON encode %v: %v", backupManifest, err)
}
if _, err := wc.Write([]byte(data)); err != nil {
return fmt.Errorf("cannot write %v: %v", backupManifest, err)
}
return nil
}
// newSnapshotFile behavior depends on the compress flag:
// - if compress is true , it compresses a single file with gzip, and
// computes the hash on the compressed version.
// - if compress is false, just symlinks and computes the hash on the file
// The source file is always left intact.
// The path of the returned SnapshotFile will be relative
// to root.
func newSnapshotFile(srcPath, dstPath, root string, compress bool) (*SnapshotFile, error) {
// open the source file
srcFile, err := os.OpenFile(srcPath, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
defer srcFile.Close()
src := bufio.NewReaderSize(srcFile, 2*1024*1024)
var hash string
var size int64
if compress {
log.Infof("newSnapshotFile: starting to compress %v into %v", srcPath, dstPath)
// open the temporary destination file
dir, filePrefix := path.Split(dstPath)
dstFile, err := ioutil.TempFile(dir, filePrefix)
if err != nil {
return nil, err
}
defer func() {
// try to close and delete the file. in the
// success case, the file will already be
// closed and renamed, so all of this would
// fail anyway, no biggie
dstFile.Close()
os.Remove(dstFile.Name())
}()
dst := bufio.NewWriterSize(dstFile, 2*1024*1024)
// create the hasher and the tee on top
hasher := newHasher()
tee := io.MultiWriter(dst, hasher)
// create the gzip compression filter
gzip, err := cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED)
if err != nil {
return nil, err
}
// copy from the file to gzip to tee to output file and hasher
_, err = io.Copy(gzip, src)
if err != nil {
return nil, err
}
// close gzip to flush it
if err = gzip.Close(); err != nil {
return nil, err
}
// close dst manually to flush all buffers to disk
dst.Flush()
dstFile.Close()
hash = hasher.HashString()
// atomically move completed compressed file
err = os.Rename(dstFile.Name(), dstPath)
if err != nil {
return nil, err
}
// and get its size
fi, err := os.Stat(dstPath)
if err != nil {
return nil, err
}
size = fi.Size()
} else {
log.Infof("newSnapshotFile: starting to hash and symlinking %v to %v", srcPath, dstPath)
// get the hash
hasher := newHasher()
_, err = io.Copy(hasher, src)
if err != nil {
return nil, err
}
hash = hasher.HashString()
// do the symlink
err = os.Symlink(srcPath, dstPath)
if err != nil {
return nil, err
}
// and get the size
fi, err := os.Stat(srcPath)
if err != nil {
return nil, err
}
size = fi.Size()
}
log.Infof("clone data ready %v:%v", dstPath, hash)
relativeDst, err := filepath.Rel(root, dstPath)
if err != nil {
return nil, err
}
return &SnapshotFile{relativeDst, size, hash, ""}, nil
}