// Synchronize file buffer onto underlying storage device. func (file *DataFile) Sync() (err error) { if err = file.Buf.Unmap(); err != nil { return } file.Buf, err = gommap.Map(file.Fh) return }
// Open a data file that grows by the specified size. func OpenDataFile(path string, growth int) (file *DataFile, err error) { file = &DataFile{Path: path, Growth: growth} if file.Fh, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600); err != nil { return } var size int64 if size, err = file.Fh.Seek(0, os.SEEK_END); err != nil { return } // Ensure the file is not smaller than file growth if file.Size = int(size); file.Size < file.Growth { if err = file.EnsureSize(growth); err != nil { return } } if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { return } for i := file.Size - 1; i >= 0; i-- { if file.Buf[i] != 0 { file.Used = i + 1 break } } tdlog.Infof("%s opened: %d of %d bytes in-use", path, file.Used, file.Size) return }
// Clear the entire file and resize it to initial size. func (file *DataFile) Clear() (err error) { if err = file.Buf.Unmap(); err != nil { return } else if err = os.Truncate(file.Path, 0); err != nil { return } else if err = os.Truncate(file.Path, int64(file.Growth)); err != nil { return } else if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { return } file.Used, file.Size = 0, file.Growth tdlog.Infof("%s cleared: %d of %d bytes in-use", file.Path, file.Used, file.Size) return }
// Open the file, or create it if non-existing. func Open(name string, growth uint64) (file *File, err error) { if growth < 1 { err = errors.New(fmt.Sprintf("Growth size (%d) is too small (opening %s)", growth, name)) } file = &File{Name: name, Growth: growth} // Open file (get a handle) and determine its size if file.Fh, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0600); err != nil { return } fsize, err := file.Fh.Seek(0, os.SEEK_END) if err != nil { return } file.Size = uint64(fsize) if file.Size == 0 { // Grow the file if it appears too small file.CheckSizeAndEnsure(file.Growth) return } // Map the file into memory buffer if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { return } // Bi-sect file buffer to find out how much space in the file is actively in-use for low, mid, high := uint64(0), file.Size/2, file.Size; ; { switch { case high-mid == 1: if ConsecutiveTwenty0s(file.Buf[mid:]) { if ConsecutiveTwenty0s(file.Buf[mid-1:]) { file.UsedSize = mid - 1 } else { file.UsedSize = mid } return } file.UsedSize = high return case ConsecutiveTwenty0s(file.Buf[mid:]): high = mid mid = low + (mid-low)/2 default: low = mid mid = mid + (high-mid)/2 } } tdlog.Printf("%s has %d bytes out of %d bytes in-use", name, file.UsedSize, file.Size) return }
// Clear the entire file and resize it to initial size. func (file *DataFile) Clear() (err error) { if err = file.Close(); err != nil { return } else if err = os.Truncate(file.Path, 0); err != nil { return } else if file.Fh, err = os.OpenFile(file.Path, os.O_CREATE|os.O_RDWR, 0600); err != nil { return } else if err = file.overwriteWithZero(0, file.Growth); err != nil { return } else if file.Buf, err = gommap.Map(file.Fh); err != nil { return } file.Used, file.Size = 0, file.Growth tdlog.Infof("%s cleared: %d of %d bytes in-use", file.Path, file.Used, file.Size) return }
// Ensure there is enough room for that many bytes of data. func (file *DataFile) EnsureSize(more int) (err error) { if file.Used+more <= file.Size { return } else if file.Buf != nil { if err = file.Buf.Unmap(); err != nil { return } } if err = file.overwriteWithZero(file.Size, file.Growth); err != nil { return } else if file.Buf, err = gommap.Map(file.Fh); err != nil { return } file.Size += file.Growth tdlog.Infof("%s grown: %d -> %d bytes (%d bytes in-use)", file.Path, file.Size-file.Growth, file.Size, file.Used) return file.EnsureSize(more) }
// Ensure there is enough room for that many bytes of data. func (file *DataFile) EnsureSize(more int) (err error) { if file.Used+more <= file.Size { return } if file.Buf != nil { if err = file.Buf.Unmap(); err != nil { return } } if err = os.Truncate(file.Path, int64(file.Size+file.Growth)); err != nil { return } else if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { return } file.Size += file.Growth tdlog.Infof("%s grown: %d -> %d bytes (%d bytes in-use)", file.Path, file.Size-file.Growth, file.Size, file.Used) return file.EnsureSize(more) }
// Open a data file that grows by the specified size. func OpenDataFile(path string, growth int) (file *DataFile, err error) { file = &DataFile{Path: path, Growth: growth} if file.Fh, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600); err != nil { return } var size int64 if size, err = file.Fh.Seek(0, os.SEEK_END); err != nil { return } // Ensure the file is not smaller than file growth if file.Size = int(size); file.Size < file.Growth { if err = file.EnsureSize(growth); err != nil { return } } if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { return } // Bi-sect file buffer to find out how much space is in-use for low, mid, high := 0, file.Size/2, file.Size; ; { switch { case high-mid == 1: if LooksEmpty(file.Buf[mid:]) { if mid > 0 && LooksEmpty(file.Buf[mid-1:]) { file.Used = mid - 1 } else { file.Used = mid } return } file.Used = high return case LooksEmpty(file.Buf[mid:]): high = mid mid = low + (mid-low)/2 default: low = mid mid = mid + (high-mid)/2 } } tdlog.Infof("%s opened: %d of %d bytes in-use", path, file.Used, file.Size) return }
// Ensure that the file has enough room for more data. Grow the file if necessary. func (file *File) CheckSizeAndEnsure(more uint64) { if file.UsedSize+more <= file.Size { return } // Grow the file - unmap the file, truncate and then re-map var err error if file.Buf != nil { if err = file.Buf.Unmap(); err != nil { panic(err) } } if err = os.Truncate(file.Name, int64(file.Size+file.Growth)); err != nil { panic(err) } if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { panic(err) } file.Size += file.Growth tdlog.Printf("File %s has grown %d bytes\n", file.Name, file.Growth) file.CheckSizeAndEnsure(more) }
// Ensure that the file has enough room for more data. Grow the file if necessary. func (file *File) CheckSizeAndEnsure(more uint64) { if file.UsedSize+more <= file.Size { return } // Unmap file buffer var err error if file.Buf != nil { if err = file.Buf.Unmap(); err != nil { panic(err) } } if _, err = file.Fh.Seek(0, os.SEEK_END); err != nil { panic(err) } // Grow file size (incrementally) zeroBuf := make([]byte, FILE_GROWTH_INCREMENTAL) for i := uint64(0); i < file.Growth; i += FILE_GROWTH_INCREMENTAL { var slice []byte if i+FILE_GROWTH_INCREMENTAL > file.Growth { slice = zeroBuf[0 : i+FILE_GROWTH_INCREMENTAL-file.Growth] } else { slice = zeroBuf } if _, err = file.Fh.Write(slice); err != nil { panic(err) } } if err = file.Fh.Sync(); err != nil { panic(err) } // Re-map the (now larger) file buffer if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { panic(err) } file.Size += file.Growth tdlog.Printf("File %s has grown %d bytes\n", file.Name, file.Growth) file.CheckSizeAndEnsure(more) }
// Overwrite the file with 0s and return to its initial size. func (file *File) Clear() { var err error if err = file.Close(); err != nil { panic(err) } // Shrink to 0 size, then enlarge if err = os.Truncate(file.Name, int64(0)); err != nil { panic(err) } if err = os.Truncate(file.Name, int64(file.Growth)); err != nil { panic(err) } // Re-open and reset current size if file.Fh, err = os.OpenFile(file.Name, os.O_CREATE|os.O_RDWR, 0600); err != nil { panic(err) } if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil { panic(err) } file.UsedSize = 0 file.Size = file.Growth tdlog.Printf("File %s has been cleared, and the size is now %d", file.Name, file.Size) }