// Synchronize file buffer onto underlying storage device.
func (file *DataFile) Sync() (err error) {
if err = file.Buf.Unmap(); err != nil {
return
}
file.Buf, err = gommap.Map(file.Fh)
return
}
// Open a data file that grows by the specified size.
func OpenDataFile(path string, growth int) (file *DataFile, err error) {
file = &DataFile{Path: path, Growth: growth}
if file.Fh, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600); err != nil {
return
}
var size int64
if size, err = file.Fh.Seek(0, os.SEEK_END); err != nil {
return
}
// Ensure the file is not smaller than file growth
if file.Size = int(size); file.Size < file.Growth {
if err = file.EnsureSize(growth); err != nil {
return
}
}
if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
return
}
for i := file.Size - 1; i >= 0; i-- {
if file.Buf[i] != 0 {
file.Used = i + 1
break
}
}
tdlog.Infof("%s opened: %d of %d bytes in-use", path, file.Used, file.Size)
return
}
// Clear the entire file and resize it to initial size.
func (file *DataFile) Clear() (err error) {
if err = file.Buf.Unmap(); err != nil {
return
} else if err = os.Truncate(file.Path, 0); err != nil {
return
} else if err = os.Truncate(file.Path, int64(file.Growth)); err != nil {
return
} else if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
return
}
file.Used, file.Size = 0, file.Growth
tdlog.Infof("%s cleared: %d of %d bytes in-use", file.Path, file.Used, file.Size)
return
}
// Open the file, or create it if non-existing.
func Open(name string, growth uint64) (file *File, err error) {
if growth < 1 {
err = errors.New(fmt.Sprintf("Growth size (%d) is too small (opening %s)", growth, name))
}
file = &File{Name: name, Growth: growth}
// Open file (get a handle) and determine its size
if file.Fh, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0600); err != nil {
return
}
fsize, err := file.Fh.Seek(0, os.SEEK_END)
if err != nil {
return
}
file.Size = uint64(fsize)
if file.Size == 0 {
// Grow the file if it appears too small
file.CheckSizeAndEnsure(file.Growth)
return
}
// Map the file into memory buffer
if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
return
}
// Bi-sect file buffer to find out how much space in the file is actively in-use
for low, mid, high := uint64(0), file.Size/2, file.Size; ; {
switch {
case high-mid == 1:
if ConsecutiveTwenty0s(file.Buf[mid:]) {
if ConsecutiveTwenty0s(file.Buf[mid-1:]) {
file.UsedSize = mid - 1
} else {
file.UsedSize = mid
}
return
}
file.UsedSize = high
return
case ConsecutiveTwenty0s(file.Buf[mid:]):
high = mid
mid = low + (mid-low)/2
default:
low = mid
mid = mid + (high-mid)/2
}
}
tdlog.Printf("%s has %d bytes out of %d bytes in-use", name, file.UsedSize, file.Size)
return
}
// Clear the entire file and resize it to initial size.
func (file *DataFile) Clear() (err error) {
if err = file.Close(); err != nil {
return
} else if err = os.Truncate(file.Path, 0); err != nil {
return
} else if file.Fh, err = os.OpenFile(file.Path, os.O_CREATE|os.O_RDWR, 0600); err != nil {
return
} else if err = file.overwriteWithZero(0, file.Growth); err != nil {
return
} else if file.Buf, err = gommap.Map(file.Fh); err != nil {
return
}
file.Used, file.Size = 0, file.Growth
tdlog.Infof("%s cleared: %d of %d bytes in-use", file.Path, file.Used, file.Size)
return
}
// Ensure there is enough room for that many bytes of data.
func (file *DataFile) EnsureSize(more int) (err error) {
if file.Used+more <= file.Size {
return
} else if file.Buf != nil {
if err = file.Buf.Unmap(); err != nil {
return
}
}
if err = file.overwriteWithZero(file.Size, file.Growth); err != nil {
return
} else if file.Buf, err = gommap.Map(file.Fh); err != nil {
return
}
file.Size += file.Growth
tdlog.Infof("%s grown: %d -> %d bytes (%d bytes in-use)", file.Path, file.Size-file.Growth, file.Size, file.Used)
return file.EnsureSize(more)
}
// Ensure there is enough room for that many bytes of data.
func (file *DataFile) EnsureSize(more int) (err error) {
if file.Used+more <= file.Size {
return
}
if file.Buf != nil {
if err = file.Buf.Unmap(); err != nil {
return
}
}
if err = os.Truncate(file.Path, int64(file.Size+file.Growth)); err != nil {
return
} else if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
return
}
file.Size += file.Growth
tdlog.Infof("%s grown: %d -> %d bytes (%d bytes in-use)", file.Path, file.Size-file.Growth, file.Size, file.Used)
return file.EnsureSize(more)
}
// Open a data file that grows by the specified size.
func OpenDataFile(path string, growth int) (file *DataFile, err error) {
file = &DataFile{Path: path, Growth: growth}
if file.Fh, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600); err != nil {
return
}
var size int64
if size, err = file.Fh.Seek(0, os.SEEK_END); err != nil {
return
}
// Ensure the file is not smaller than file growth
if file.Size = int(size); file.Size < file.Growth {
if err = file.EnsureSize(growth); err != nil {
return
}
}
if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
return
}
// Bi-sect file buffer to find out how much space is in-use
for low, mid, high := 0, file.Size/2, file.Size; ; {
switch {
case high-mid == 1:
if LooksEmpty(file.Buf[mid:]) {
if mid > 0 && LooksEmpty(file.Buf[mid-1:]) {
file.Used = mid - 1
} else {
file.Used = mid
}
return
}
file.Used = high
return
case LooksEmpty(file.Buf[mid:]):
high = mid
mid = low + (mid-low)/2
default:
low = mid
mid = mid + (high-mid)/2
}
}
tdlog.Infof("%s opened: %d of %d bytes in-use", path, file.Used, file.Size)
return
}
// Ensure that the file has enough room for more data. Grow the file if necessary.
func (file *File) CheckSizeAndEnsure(more uint64) {
if file.UsedSize+more <= file.Size {
return
}
// Grow the file - unmap the file, truncate and then re-map
var err error
if file.Buf != nil {
if err = file.Buf.Unmap(); err != nil {
panic(err)
}
}
if err = os.Truncate(file.Name, int64(file.Size+file.Growth)); err != nil {
panic(err)
}
if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
panic(err)
}
file.Size += file.Growth
tdlog.Printf("File %s has grown %d bytes\n", file.Name, file.Growth)
file.CheckSizeAndEnsure(more)
}
// Ensure that the file has enough room for more data. Grow the file if necessary.
func (file *File) CheckSizeAndEnsure(more uint64) {
if file.UsedSize+more <= file.Size {
return
}
// Unmap file buffer
var err error
if file.Buf != nil {
if err = file.Buf.Unmap(); err != nil {
panic(err)
}
}
if _, err = file.Fh.Seek(0, os.SEEK_END); err != nil {
panic(err)
}
// Grow file size (incrementally)
zeroBuf := make([]byte, FILE_GROWTH_INCREMENTAL)
for i := uint64(0); i < file.Growth; i += FILE_GROWTH_INCREMENTAL {
var slice []byte
if i+FILE_GROWTH_INCREMENTAL > file.Growth {
slice = zeroBuf[0 : i+FILE_GROWTH_INCREMENTAL-file.Growth]
} else {
slice = zeroBuf
}
if _, err = file.Fh.Write(slice); err != nil {
panic(err)
}
}
if err = file.Fh.Sync(); err != nil {
panic(err)
}
// Re-map the (now larger) file buffer
if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
panic(err)
}
file.Size += file.Growth
tdlog.Printf("File %s has grown %d bytes\n", file.Name, file.Growth)
file.CheckSizeAndEnsure(more)
}
// Overwrite the file with 0s and return to its initial size.
func (file *File) Clear() {
var err error
if err = file.Close(); err != nil {
panic(err)
}
// Shrink to 0 size, then enlarge
if err = os.Truncate(file.Name, int64(0)); err != nil {
panic(err)
}
if err = os.Truncate(file.Name, int64(file.Growth)); err != nil {
panic(err)
}
// Re-open and reset current size
if file.Fh, err = os.OpenFile(file.Name, os.O_CREATE|os.O_RDWR, 0600); err != nil {
panic(err)
}
if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
panic(err)
}
file.UsedSize = 0
file.Size = file.Growth
tdlog.Printf("File %s has been cleared, and the size is now %d", file.Name, file.Size)
}