Ejemplo n.º 1
0
// Copy - copy data from source to destination
func (f *fsClient) Copy(source string, size int64, progress io.Reader) *probe.Error {
	// Don't use f.Get() f.Put() directly. Instead use readFile and createFile
	destination := f.PathURL.Path
	if destination == source { // Cannot copy file into itself
		return errOverWriteNotAllowed(destination).Trace(destination)
	}
	rc, e := readFile(source)
	if e != nil {
		err := f.toClientError(e, destination)
		return err.Trace(destination)
	}
	defer rc.Close()
	wc, e := createFile(destination)
	if e != nil {
		err := f.toClientError(e, destination)
		return err.Trace(destination)
	}
	defer wc.Close()
	reader := hookreader.NewHook(rc, progress)
	// Perform copy
	n, _ := io.CopyN(wc, reader, size) // e == nil only if n != size
	// Only check size related errors if size is positive
	if size > 0 {
		if n < size { // Unexpected early EOF
			return probe.NewError(UnexpectedEOF{
				TotalSize:    size,
				TotalWritten: n,
			})
		}
		if n > size { // Unexpected ExcessRead
			return probe.NewError(UnexpectedExcessRead{
				TotalSize:    size,
				TotalWritten: n,
			})
		}
	}
	return nil
}
Ejemplo n.º 2
0
// Put - create a new file.
func (f *fsClient) Put(reader io.Reader, size int64, contentType string, progress io.Reader) (int64, *probe.Error) {
	// ContentType is not handled on purpose.
	// For filesystem this is a redundant information.

	// Extract dir name.
	objectDir, _ := filepath.Split(f.PathURL.Path)
	objectPath := f.PathURL.Path

	// Verify if destination already exists.
	st, e := os.Stat(objectPath)
	if e == nil {
		// If the destination exists and is not a regular file.
		if !st.Mode().IsRegular() {
			return 0, probe.NewError(PathIsNotRegular{
				Path: objectPath,
			})
		}
	}

	// Proceed if file does not exist. return for all other errors.
	if e != nil {
		if !os.IsNotExist(e) {
			return 0, probe.NewError(e)
		}
	}

	// Write to a temporary file "object.part.mc" before commit.
	objectPartPath := objectPath + partSuffix
	if objectDir != "" {
		// Create any missing top level directories.
		if e = os.MkdirAll(objectDir, 0700); e != nil {
			err := f.toClientError(e, f.PathURL.Path)
			return 0, err.Trace(f.PathURL.Path)
		}
	}

	// If exists, open in append mode. If not create it the part file.
	partFile, e := os.OpenFile(objectPartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if e != nil {
		err := f.toClientError(e, f.PathURL.Path)
		return 0, err.Trace(f.PathURL.Path)
	}

	// Get stat to get the current size.
	partSt, e := partFile.Stat()
	if e != nil {
		err := f.toClientError(e, objectPartPath)
		return 0, err.Trace(objectPartPath)
	}

	var totalWritten int64
	// Current file offset.
	var currentOffset = partSt.Size()

	// Use ReadAt() capability when reader implements it, but also avoid it in two cases:
	// *) reader represents a standard input/output stream since they return illegal seek error when ReadAt() is invoked
	// *) we know in advance that reader will provide zero length data
	if readerAt, ok := reader.(io.ReaderAt); ok && !isStdIO(reader) && size > 0 {
		// Notify the progress bar if any till current size.
		if progress != nil {
			if _, e = io.CopyN(ioutil.Discard, progress, currentOffset); e != nil {
				return 0, probe.NewError(e)
			}
		}

		// Allocate buffer of 10MiB once.
		readAtBuffer := make([]byte, 10*1024*1024)

		// Loop through all offsets on incoming io.ReaderAt and write
		// to the destination.
		for currentOffset < size {
			readAtSize, re := readerAt.ReadAt(readAtBuffer, currentOffset)
			if re != nil && re != io.EOF {
				// For any errors other than io.EOF, we return error
				// and breakout.
				err := f.toClientError(re, objectPartPath)
				return 0, err.Trace(objectPartPath)
			}
			writtenSize, we := partFile.Write(readAtBuffer[:readAtSize])
			if we != nil {
				err := f.toClientError(we, objectPartPath)
				return 0, err.Trace(objectPartPath)
			}
			// read size and subsequent write differ, a possible
			// corruption return here.
			if readAtSize != writtenSize {
				// Unexpected write (less data was written than expected).
				return 0, probe.NewError(UnexpectedShortWrite{
					InputSize: readAtSize,
					WriteSize: writtenSize,
				})
			}
			// Notify the progress bar if any for written size.
			if progress != nil {
				if _, e = io.CopyN(ioutil.Discard, progress, int64(writtenSize)); e != nil {
					return totalWritten, probe.NewError(e)
				}
			}
			currentOffset += int64(writtenSize)
			// Once we see io.EOF we break out of the loop.
			if re == io.EOF {
				break
			}
		}
		// Save currently copied total into totalWritten.
		totalWritten = currentOffset
	} else {
		reader = hookreader.NewHook(reader, progress)
		// Discard bytes until currentOffset.
		if _, e = io.CopyN(ioutil.Discard, reader, currentOffset); e != nil {
			return 0, probe.NewError(e)
		}
		var n int64
		n, e = io.Copy(partFile, reader)
		if e != nil {
			return 0, probe.NewError(e)
		}
		// Save currently copied total into totalWritten.
		totalWritten = n + currentOffset
	}

	// Close the input reader as well, if possible.
	closer, ok := reader.(io.Closer)
	if ok {
		if e = closer.Close(); e != nil {
			return totalWritten, probe.NewError(e)
		}
	}

	// Close the file before rename.
	if e = partFile.Close(); e != nil {
		return totalWritten, probe.NewError(e)
	}

	// Following verification is needed only for input size greater than '0'.
	if size > 0 {
		// Unexpected EOF reached (less data was written than expected).
		if totalWritten < size {
			return totalWritten, probe.NewError(UnexpectedEOF{
				TotalSize:    size,
				TotalWritten: totalWritten,
			})
		}
		// Unexpected ExcessRead (more data was written than expected).
		if totalWritten > size {
			return totalWritten, probe.NewError(UnexpectedExcessRead{
				TotalSize:    size,
				TotalWritten: totalWritten,
			})
		}
	}
	// Safely completed put. Now commit by renaming to actual filename.
	if e = os.Rename(objectPartPath, objectPath); e != nil {
		err := f.toClientError(e, objectPath)
		return totalWritten, err.Trace(objectPartPath, objectPath)
	}
	return totalWritten, nil
}