func TestFallocate(t *testing.T) {
ts := NewTestCase(t)
defer ts.Cleanup()
if ts.state.KernelSettings().Minor < 19 {
t.Log("FUSE does not support Fallocate.")
return
}
rwFile, err := os.OpenFile(ts.mnt+"/file", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
t.Fatalf("OpenFile failed: %v", err)
}
defer rwFile.Close()
err = syscall.Fallocate(int(rwFile.Fd()), 0, 1024, 4096)
if err != nil {
t.Fatalf("FUSE Fallocate failed: %v", err)
}
fi, err := os.Lstat(ts.orig + "/file")
if err != nil {
t.Fatalf("Lstat failed: %v", err)
}
if fi.Size() < (1024 + 4096) {
t.Fatalf("fallocate should have changed file size. Got %d bytes",
fi.Size())
}
}
func New(capacity int, filename string) (*Buffer, error) {
if _, err := os.Stat(filename); !os.IsNotExist(err) {
return nil, ErrFileExists
}
f, err := os.Create(filename)
if err != nil {
return nil, err
}
defer f.Close()
fsize := capacity + metadata
if err := syscall.Fallocate(int(f.Fd()), 0, 0, int64(fsize)); err != nil {
return nil, err
}
data, err := syscall.Mmap(
int(f.Fd()), 0, fsize,
syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED,
)
// TODO(ashish): Call msync periodically.
if err != nil {
return nil, err
}
b := &Buffer{
capacity: uint64(capacity), // since it's used with uint64s a lot more
filename: f.Name(),
data: data,
}
b.updateMeta()
return b, nil
}
func fallocate(f *os.File, sz int64) error {
err := syscall.Fallocate(int(f.Fd()), 0, 0, sz)
if err == syscall.ENOTSUP {
return f.Truncate(sz)
}
return err
}
// puncHoleLinux punches a hole into the given file starting at offset,
// measuring "size" bytes
func punchHoleLinux(file *os.File, offset int64, size int64) error {
err := syscall.Fallocate(int(file.Fd()),
falloc_fl_punch_hole|falloc_fl_keep_size,
offset, size)
if err == syscall.ENOSYS || err == syscall.EOPNOTSUPP {
return errNoPunch
}
return err
}
func (f *LoopbackFile) Allocate(off uint64, sz uint64, mode uint32) Status {
f.lock.Lock()
err := syscall.Fallocate(int(f.File.Fd()), mode, int64(off), int64(sz))
f.lock.Unlock()
if err != nil {
return ToStatus(err)
}
return OK
}
// fallocateRetry - syscall.Fallocate() with retry for EINTR.
func fallocateRetry(fd int, mode uint32, off int64, len int64) (err error) {
for {
err = syscall.Fallocate(fd, mode, off, len)
if err == syscall.EINTR {
continue
}
return err
}
}
func (f *AzukiFile) Allocate(off uint64, sz uint64, mode uint32) fuse.Status {
f.lock.Lock()
err := syscall.Fallocate(int(f.File.Fd()), mode, int64(off), int64(sz))
f.lock.Unlock()
if err != nil {
return fuse.ToStatus(err)
}
go event.Notify(event.Fallocate, f.File.Name())
return fuse.OK
}
// prealloc - preallocate space without changing the file size. This prevents
// us from running out of space in the middle of an operation.
func prealloc(fd int, off int64, len int64) (err error) {
for {
err = syscall.Fallocate(fd, FALLOC_FL_KEEP_SIZE, off, len)
if err == syscall.EINTR {
// fallocate, like many syscalls, can return EINTR. This is not an
// error and just signifies that the operation was interrupted by a
// signal and we should try again.
continue
}
return err
}
}
func preallocFixed(f *os.File, sizeInBytes int64) error {
// use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE
err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes)
if err != nil {
errno, ok := err.(syscall.Errno)
// treat not supported as nil error
if ok && (errno == syscall.ENOTSUP || errno == syscall.EINTR) {
return nil
}
}
return err
}
func createBackingFile(backing string) error {
flags := syscall.O_RDWR | syscall.O_CREAT | syscall.O_EXCL
file, err := os.OpenFile(backing, flags, 0755)
if err != nil {
return fmt.Errorf("Failed to create backing file %s: %s.", backing, err)
}
err = syscall.Fallocate(int(file.Fd()), 0, 0, fileSizeInBytes)
if err != nil {
return fmt.Errorf("Failed to allocate %s: %s.", backing, err)
}
return nil
}
func preallocExtend(f *os.File, sizeInBytes int64) error {
// use mode = 0 to change size
err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes)
if err != nil {
errno, ok := err.(syscall.Errno)
// treat not support as nil error
if ok && errno == syscall.ENOTSUP {
return preallocExtendTrunc(f, sizeInBytes)
}
}
return err
}
func preallocExtend(f *os.File, sizeInBytes int64) error {
// use mode = 0 to change size
err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes)
if err != nil {
errno, ok := err.(syscall.Errno)
// not supported; fallback
// fallocate EINTRs frequently in some enviroments; fallback
if ok && (errno == syscall.ENOTSUP || errno == syscall.EINTR) {
return preallocExtendTrunc(f, sizeInBytes)
}
}
return err
}
// Preallocate tries to allocate the space for given
// file. This operation is only supported on linux by a
// few filesystems (btrfs, ext4, etc.).
// If the operation is unsupported, no error will be returned.
// Otherwise, the error encountered will be returned.
func Preallocate(f *os.File, sizeInBytes int) error {
// use mode = 1 to keep size
// see FALLOC_FL_KEEP_SIZE
err := syscall.Fallocate(int(f.Fd()), 1, 0, int64(sizeInBytes))
if err != nil {
errno, ok := err.(syscall.Errno)
// treat not support as nil error
if ok && errno == syscall.ENOTSUP {
return nil
}
return err
}
return nil
}
func main() {
flag.Usage = usage
flag.Parse()
if flag.NArg() < 1 {
usage()
}
status := 0
for _, name := range flag.Args() {
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0644)
if ek(err) {
continue
}
ek(syscall.Fallocate(int(f.Fd()), 0, *offset, *size))
}
os.Exit(status)
}
// NewOutputValueFile creates an entity implmenting the OutputValueFile interface
func NewOutputValueFile(fileSpaceInfo tools.FileSpaceInfo) (OutputValueFile, error) {
var valueFile outputValueFile
var err error
valueFile.creationTime = tools.Timestamp()
valueFile.md5Sum = md5.New()
valueFile.collectionIDSet = make(map[uint32]struct{})
repositoryPath := os.Getenv("NIMBUSIO_REPOSITORY_PATH")
if valueFile.spaceID, err = fileSpaceInfo.FindMaxAvailSpaceID(tools.FileSpaceJournal); err != nil {
return nil, err
}
if err = valueFile.insertValueFileRow(); err != nil {
return nil, err
}
valueFile.filePath = tools.ComputeValueFilePath(repositoryPath, valueFile.spaceID,
valueFile.valueFileID)
fog.Debug("NewOutputValueFile %s", valueFile.filePath)
dirPath := path.Dir(valueFile.filePath)
if err = os.MkdirAll(dirPath, os.ModeDir|0755); err != nil {
return nil, fmt.Errorf("os.MkdirAll(%s...", err)
}
valueFile.fileHandle, err = os.Create(valueFile.filePath)
if err != nil {
return nil, fmt.Errorf("os.Create(%s) %s", valueFile.filePath, err)
}
err = syscall.Fallocate(int(valueFile.fileHandle.Fd()), 0, 0,
int64(MaxValueFileSize))
if err != nil {
return nil, fmt.Errorf("Fallocate failed %s", err)
}
valueFile.enableFsync = os.Getenv("NIMBUSIO_ENABLE_FSYNC") == "1"
fog.Info("NewOutputValueFile: NIMBUSIO_ENABLE_FSYNC = %t", valueFile.enableFsync)
return &valueFile, nil
}
// PunchHole in a sparse file, preserve file size
func PunchHole(file *os.File, hole Interval) error {
fd := int(file.Fd())
mode := fallocFlPunchHole | fallocFlKeepSize
return syscall.Fallocate(fd, mode, hole.Begin, hole.Len())
}
func Fallocate(fd uintptr, mode uint32, off int64, len int64) (err error) {
return syscall.Fallocate(int(fd), mode, off, len)
}
func (server ObjectServer) ObjPutHandler(writer *SwiftWriter, request *SwiftRequest, vars map[string]string) {
outHeaders := writer.Header()
hashDir, err := ObjHashDir(vars, server)
if err != nil {
http.Error(writer, "Insufficent Storage", 507)
return
}
if inm := request.Header.Get("If-None-Match"); inm == "*" {
dataFile := PrimaryFile(hashDir)
if dataFile != "" && !strings.HasSuffix(dataFile, ".ts") {
http.Error(writer, http.StatusText(http.StatusPreconditionFailed), http.StatusPreconditionFailed)
return
}
}
if os.MkdirAll(hashDir, 0770) != nil || os.MkdirAll(ObjTempDir(vars, server), 0770) != nil {
http.Error(writer, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
fileName := fmt.Sprintf("%s/%s.data", hashDir, request.Header.Get("X-Timestamp"))
tempFile, err := ioutil.TempFile(ObjTempDir(vars, server), "PUT")
if err != nil {
http.Error(writer, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
if contentLength, err := strconv.ParseInt(request.Header.Get("Content-Type"), 10, 64); err == nil {
syscall.Fallocate(int(tempFile.Fd()), 1, 0, contentLength)
}
defer tempFile.Close()
defer os.RemoveAll(tempFile.Name())
metadata := make(map[string]interface{})
metadata["name"] = fmt.Sprintf("/%s/%s/%s", vars["account"], vars["container"], vars["obj"])
metadata["X-Timestamp"] = request.Header.Get("X-Timestamp")
metadata["Content-Type"] = request.Header.Get("Content-Type")
for key := range request.Header {
if allowed, ok := server.allowedHeaders[key]; (ok && allowed) || strings.HasPrefix(key, "X-Object-Meta-") {
metadata[key] = request.Header.Get(key)
}
}
hash := md5.New()
totalSize, err := io.Copy(hash, io.TeeReader(request.Body, tempFile))
if err != nil {
fmt.Printf("ERROR: %s\n", err)
return
}
go DropBufferCache(int(tempFile.Fd()), totalSize)
metadata["Content-Length"] = strconv.FormatInt(totalSize, 10)
metadata["ETag"] = fmt.Sprintf("%x", hash.Sum(nil))
requestEtag := request.Header.Get("ETag")
if requestEtag != "" && requestEtag != metadata["ETag"].(string) {
http.Error(writer, "Unprocessable Entity", 422)
return
}
outHeaders.Set("ETag", metadata["ETag"].(string))
WriteMetadata(int(tempFile.Fd()), metadata)
if !server.disableFsync {
tempFile.Sync()
}
os.Rename(tempFile.Name(), fileName)
finalize := func() {
UpdateContainer(metadata, request, vars)
if request.Header.Get("X-Delete-At") != "" || request.Header.Get("X-Delete-After") != "" {
UpdateDeleteAt(request, vars, metadata)
}
CleanupHashDir(hashDir)
InvalidateHash(hashDir, !server.disableFsync)
}
if server.asyncFinalize {
go finalize()
} else {
finalize()
}
http.Error(writer, http.StatusText(http.StatusCreated), http.StatusCreated)
}
func (c *Container) falloc() (err error) {
return syscall.Fallocate(int(c.f.Fd()), 0, 0, c.Size)
}
// Fallocate : allocate using fallocate
func (f FiemapFile) Fallocate(offset int64, length int64) error {
return syscall.Fallocate(int(f.Fd()), 0, offset, length)
}
func Preallocate(f *os.File, sizeInBytes int) error {
// use mode = 1 to keep size
// see FALLOC_FL_KEEP_SIZE
return syscall.Fallocate(int(f.Fd()), 1, 0, int64(sizeInBytes))
}
// Fallocate uses the linux Fallocate syscall, which helps us to be
// sure that subsequent writes on a file just created will not fail,
// in addition, file allocation will be contigous on the disk
func Fallocate(fd int, offset int64, len int64) error {
return syscall.Fallocate(fd,
1, // FALLOC_FL_KEEP_SIZE
offset,
len)
}
// PunchHole : punch hole using fallocate
func (f FiemapFile) PunchHole(offset int64, length int64) error {
return syscall.Fallocate(int(f.Fd()), FALLOC_FL_KEEP_SIZE|FALLOC_FL_PUNCH_HOLE, offset, length)
}