// Copy - copy data from source to destination
func (f *fsClient) Copy(source string, size int64, progress io.Reader) *probe.Error {
// Don't use f.Get() f.Put() directly. Instead use readFile and createFile
destination := f.PathURL.Path
if destination == source { // Cannot copy file into itself
return errOverWriteNotAllowed(destination).Trace(destination)
}
rc, e := readFile(source)
if e != nil {
err := f.toClientError(e, destination)
return err.Trace(destination)
}
defer rc.Close()
wc, e := createFile(destination)
if e != nil {
err := f.toClientError(e, destination)
return err.Trace(destination)
}
defer wc.Close()
reader := hookreader.NewHook(rc, progress)
// Perform copy
n, _ := io.CopyN(wc, reader, size) // e == nil only if n != size
// Only check size related errors if size is positive
if size > 0 {
if n < size { // Unexpected early EOF
return probe.NewError(UnexpectedEOF{
TotalSize: size,
TotalWritten: n,
})
}
if n > size { // Unexpected ExcessRead
return probe.NewError(UnexpectedExcessRead{
TotalSize: size,
TotalWritten: n,
})
}
}
return nil
}
// Put - create a new file.
func (f *fsClient) Put(reader io.Reader, size int64, contentType string, progress io.Reader) (int64, *probe.Error) {
// ContentType is not handled on purpose.
// For filesystem this is a redundant information.
// Extract dir name.
objectDir, _ := filepath.Split(f.PathURL.Path)
objectPath := f.PathURL.Path
// Verify if destination already exists.
st, e := os.Stat(objectPath)
if e == nil {
// If the destination exists and is not a regular file.
if !st.Mode().IsRegular() {
return 0, probe.NewError(PathIsNotRegular{
Path: objectPath,
})
}
}
// Proceed if file does not exist. return for all other errors.
if e != nil {
if !os.IsNotExist(e) {
return 0, probe.NewError(e)
}
}
// Write to a temporary file "object.part.mc" before commit.
objectPartPath := objectPath + partSuffix
if objectDir != "" {
// Create any missing top level directories.
if e = os.MkdirAll(objectDir, 0700); e != nil {
err := f.toClientError(e, f.PathURL.Path)
return 0, err.Trace(f.PathURL.Path)
}
}
// If exists, open in append mode. If not create it the part file.
partFile, e := os.OpenFile(objectPartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if e != nil {
err := f.toClientError(e, f.PathURL.Path)
return 0, err.Trace(f.PathURL.Path)
}
// Get stat to get the current size.
partSt, e := partFile.Stat()
if e != nil {
err := f.toClientError(e, objectPartPath)
return 0, err.Trace(objectPartPath)
}
var totalWritten int64
// Current file offset.
var currentOffset = partSt.Size()
// Use ReadAt() capability when reader implements it, but also avoid it in two cases:
// *) reader represents a standard input/output stream since they return illegal seek error when ReadAt() is invoked
// *) we know in advance that reader will provide zero length data
if readerAt, ok := reader.(io.ReaderAt); ok && !isStdIO(reader) && size > 0 {
// Notify the progress bar if any till current size.
if progress != nil {
if _, e = io.CopyN(ioutil.Discard, progress, currentOffset); e != nil {
return 0, probe.NewError(e)
}
}
// Allocate buffer of 10MiB once.
readAtBuffer := make([]byte, 10*1024*1024)
// Loop through all offsets on incoming io.ReaderAt and write
// to the destination.
for currentOffset < size {
readAtSize, re := readerAt.ReadAt(readAtBuffer, currentOffset)
if re != nil && re != io.EOF {
// For any errors other than io.EOF, we return error
// and breakout.
err := f.toClientError(re, objectPartPath)
return 0, err.Trace(objectPartPath)
}
writtenSize, we := partFile.Write(readAtBuffer[:readAtSize])
if we != nil {
err := f.toClientError(we, objectPartPath)
return 0, err.Trace(objectPartPath)
}
// read size and subsequent write differ, a possible
// corruption return here.
if readAtSize != writtenSize {
// Unexpected write (less data was written than expected).
return 0, probe.NewError(UnexpectedShortWrite{
InputSize: readAtSize,
WriteSize: writtenSize,
})
}
// Notify the progress bar if any for written size.
if progress != nil {
if _, e = io.CopyN(ioutil.Discard, progress, int64(writtenSize)); e != nil {
return totalWritten, probe.NewError(e)
}
}
currentOffset += int64(writtenSize)
// Once we see io.EOF we break out of the loop.
if re == io.EOF {
break
}
}
// Save currently copied total into totalWritten.
totalWritten = currentOffset
} else {
reader = hookreader.NewHook(reader, progress)
// Discard bytes until currentOffset.
if _, e = io.CopyN(ioutil.Discard, reader, currentOffset); e != nil {
return 0, probe.NewError(e)
}
var n int64
n, e = io.Copy(partFile, reader)
if e != nil {
return 0, probe.NewError(e)
}
// Save currently copied total into totalWritten.
totalWritten = n + currentOffset
}
// Close the input reader as well, if possible.
closer, ok := reader.(io.Closer)
if ok {
if e = closer.Close(); e != nil {
return totalWritten, probe.NewError(e)
}
}
// Close the file before rename.
if e = partFile.Close(); e != nil {
return totalWritten, probe.NewError(e)
}
// Following verification is needed only for input size greater than '0'.
if size > 0 {
// Unexpected EOF reached (less data was written than expected).
if totalWritten < size {
return totalWritten, probe.NewError(UnexpectedEOF{
TotalSize: size,
TotalWritten: totalWritten,
})
}
// Unexpected ExcessRead (more data was written than expected).
if totalWritten > size {
return totalWritten, probe.NewError(UnexpectedExcessRead{
TotalSize: size,
TotalWritten: totalWritten,
})
}
}
// Safely completed put. Now commit by renaming to actual filename.
if e = os.Rename(objectPartPath, objectPath); e != nil {
err := f.toClientError(e, objectPath)
return totalWritten, err.Trace(objectPartPath, objectPath)
}
return totalWritten, nil
}