// openLast opens the last wal file for read and write. func openLast(dirpath string) (*fileutil.LockedFile, error) { names, err := readWalNames(dirpath) if err != nil { return nil, err } last := path.Join(dirpath, names[len(names)-1]) return fileutil.LockFile(last, os.O_RDWR, fileutil.PrivateFileMode) }
// Create creates a WAL ready for appending records. The given metadata is // recorded at the head of each WAL file, and can be retrieved with ReadAll. func Create(dirpath string, metadata []byte) (*WAL, error) { if Exist(dirpath) { return nil, os.ErrExist } // keep temporary wal directory so WAL initialization appears atomic tmpdirpath := path.Clean(dirpath) + ".tmp" if fileutil.Exist(tmpdirpath) { if err := os.RemoveAll(tmpdirpath); err != nil { return nil, err } } if err := os.MkdirAll(tmpdirpath, fileutil.PrivateDirMode); err != nil { return nil, err } p := path.Join(tmpdirpath, walName(0, 0)) f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode) if err != nil { return nil, err } if _, err := f.Seek(0, os.SEEK_END); err != nil { return nil, err } if err := fileutil.Preallocate(f.File, segmentSizeBytes, true); err != nil { return nil, err } w := &WAL{ dir: dirpath, metadata: metadata, encoder: newEncoder(f, 0), } w.locks = append(w.locks, f) if err := w.saveCrc(0); err != nil { return nil, err } if err := w.encoder.encode(&walpb.Record{Type: metadataType, Data: metadata}); err != nil { return nil, err } if err := w.SaveSnapshot(walpb.Snapshot{}); err != nil { return nil, err } if err := os.RemoveAll(dirpath); err != nil { return nil, err } if err := os.Rename(tmpdirpath, dirpath); err != nil { return nil, err } w.fp = newFilePipeline(w.dir, segmentSizeBytes) return w, nil }
// openLast opens the last wal file for read and write. func openLast(dirpath string) (*fileutil.LockedFile, error) { names, err := fileutil.ReadDir(dirpath) if err != nil { return nil, err } names = checkWalNames(names) if len(names) == 0 { return nil, ErrFileNotFound } last := path.Join(dirpath, names[len(names)-1]) return fileutil.LockFile(last, os.O_RDWR, 0600) }
func (fp *filePipeline) alloc() (f *fileutil.LockedFile, err error) { fpath := path.Join(fp.dir, fmt.Sprintf("%d.tmp", fp.count)) if f, err = fileutil.LockFile(fpath, os.O_CREATE|os.O_WRONLY, 0600); err != nil { return nil, err } if err = fileutil.Preallocate(f.File, fp.size, true); err != nil { plog.Errorf("failed to allocate space when creating new wal file (%v)", err) f.Close() return nil, err } fp.count++ return f, nil }
func (fp *filePipeline) alloc() (f *fileutil.LockedFile, err error) { // count % 2 so this file isn't the same as the one last published fpath := path.Join(fp.dir, fmt.Sprintf("%d.tmp", fp.count%2)) if f, err = fileutil.LockFile(fpath, os.O_CREATE|os.O_WRONLY, fileutil.PrivateFileMode); err != nil { return nil, err } if err = fileutil.Preallocate(f.File, fp.size, true); err != nil { plog.Errorf("failed to allocate space when creating new wal file (%v)", err) f.Close() return nil, err } fp.count++ return f, nil }
// Retrieves the path for the cache file. It checks the following directories // in this specific order: // 1. $HOME/.cache // 2. /tmp // It will try to open (or create if it doesn't exist) the cache file in each // directory until it finds a directory that is accessible. func cachePath() *os.File { candidates := []string{filepath.Join(os.Getenv("HOME"), ".cache"), "/tmp"} for _, dir := range candidates { dirs := strings.Split(dir, ":") for _, d := range dirs { name := filepath.Join(d, cacheName) lock, err := fileutil.LockFile(name, os.O_RDWR|os.O_CREATE, 0666) if err == nil { return lock.File } } } return nil }
// Create creates a WAL ready for appending records. The given metadata is // recorded at the head of each WAL file, and can be retrieved with ReadAll. func Create(dirpath string, metadata []byte) (*WAL, error) { if Exist(dirpath) { return nil, os.ErrExist } if err := os.MkdirAll(dirpath, privateDirMode); err != nil { return nil, err } p := path.Join(dirpath, walName(0, 0)) f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return nil, err } if _, err := f.Seek(0, os.SEEK_END); err != nil { return nil, err } if err := fileutil.Preallocate(f.File, segmentSizeBytes, true); err != nil { return nil, err } w := &WAL{ dir: dirpath, metadata: metadata, encoder: newEncoder(f, 0), fp: newFilePipeline(dirpath, segmentSizeBytes), } w.locks = append(w.locks, f) if err := w.saveCrc(0); err != nil { return nil, err } if err := w.encoder.encode(&walpb.Record{Type: metadataType, Data: metadata}); err != nil { return nil, err } if err := w.SaveSnapshot(walpb.Snapshot{}); err != nil { return nil, err } return w, nil }
// Writes all the cached data back to the cache file. This is needed because // functions like `inSUSE` only write to memory. Therefore, once you're done // with this instance, you should call this function to keep everything synced. func (cd *cachedData) flush() { if !cd.Valid { // Silently fail, the user has probably already been notified about it. return } file, err := fileutil.LockFile(cd.Path, os.O_RDWR, 0666) if err != nil { log.Printf("Cannot write to the cache file: %v", err) return } defer file.Close() _, err = file.Stat() if err != nil { log.Printf("Cannot stat file: %v", err) return } // Read cache from file (again) before writing to it, otherwise the // cache will possibly be inconsistent. oldCache := cd.readCache(file) // Merge the old and "new" cache, and remove duplicates. cd.Suse = append(cd.Suse, oldCache.Suse...) cd.Other = append(cd.Other, oldCache.Other...) cd.Outdated = append(cd.Outdated, oldCache.Outdated...) cd.Suse = removeDuplicates(cd.Suse) cd.Other = removeDuplicates(cd.Other) cd.Outdated = removeDuplicates(cd.Outdated) // Clear file content. file.Seek(0, 0) file.Truncate(0) enc := json.NewEncoder(file) _ = enc.Encode(cd) }
// cut closes current file written and creates a new one ready to append. // cut first creates a temp wal file and writes necessary headers into it. // Then cut atomically rename temp wal file to a wal file. func (w *WAL) cut() error { // close old wal file; truncate to avoid wasting space if an early cut off, serr := w.tail().Seek(0, os.SEEK_CUR) if serr != nil { return serr } if err := w.tail().Truncate(off); err != nil { return err } if err := w.sync(); err != nil { return err } fpath := path.Join(w.dir, walName(w.seq()+1, w.enti+1)) // create a temp wal file with name sequence + 1, or truncate the existing one newTail, err := w.fp.Open() if err != nil { return err } // update writer and save the previous crc w.locks = append(w.locks, newTail) prevCrc := w.encoder.crc.Sum32() w.encoder = newEncoder(w.tail(), prevCrc) if err = w.saveCrc(prevCrc); err != nil { return err } if err = w.encoder.encode(&walpb.Record{Type: metadataType, Data: w.metadata}); err != nil { return err } if err = w.saveState(&w.state); err != nil { return err } // atomically move temp wal file to wal file if err = w.sync(); err != nil { return err } off, err = w.tail().Seek(0, os.SEEK_CUR) if err != nil { return err } if err = os.Rename(newTail.Name(), fpath); err != nil { return err } newTail.Close() if newTail, err = fileutil.LockFile(fpath, os.O_WRONLY, fileutil.PrivateFileMode); err != nil { return err } if _, err = newTail.Seek(off, os.SEEK_SET); err != nil { return err } w.locks[len(w.locks)-1] = newTail prevCrc = w.encoder.crc.Sum32() w.encoder = newEncoder(w.tail(), prevCrc) plog.Infof("segmented wal file %v is created", fpath) return nil }
// Create creates a WAL ready for appending records. The given metadata is // recorded at the head of each WAL file, and can be retrieved with ReadAll. func Create(dirpath string, metadata []byte) (*WAL, error) { if Exist(dirpath) { return nil, os.ErrExist } // keep temporary wal directory so WAL initialization appears atomic tmpdirpath := path.Clean(dirpath) + ".tmp" if fileutil.Exist(tmpdirpath) { if err := os.RemoveAll(tmpdirpath); err != nil { return nil, err } } if err := fileutil.CreateDirAll(tmpdirpath); err != nil { return nil, err } p := path.Join(tmpdirpath, walName(0, 0)) f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode) if err != nil { return nil, err } if _, err = f.Seek(0, os.SEEK_END); err != nil { return nil, err } if err = fileutil.Preallocate(f.File, SegmentSizeBytes, true); err != nil { return nil, err } w := &WAL{ dir: dirpath, metadata: metadata, encoder: newEncoder(f, 0), } w.locks = append(w.locks, f) if err = w.saveCrc(0); err != nil { return nil, err } if err = w.encoder.encode(&walpb.Record{Type: metadataType, Data: metadata}); err != nil { return nil, err } if err = w.SaveSnapshot(walpb.Snapshot{}); err != nil { return nil, err } if w, err = w.renameWal(tmpdirpath); err != nil { return nil, err } // directory was renamed; sync parent dir to persist rename pdir, perr := fileutil.OpenDir(path.Dir(w.dir)) if perr != nil { return nil, perr } if perr = fileutil.Fsync(pdir); perr != nil { return nil, perr } if perr = pdir.Close(); err != nil { return nil, perr } return w, nil }
// Create creates a WAL ready for appending records. The given metadata is // recorded at the head of each WAL file, and can be retrieved with ReadAll. func Create(dirpath string, metadata []byte) (*WAL, error) { if Exist(dirpath) { return nil, os.ErrExist } // keep temporary wal directory so WAL initialization appears atomic tmpdirpath := path.Clean(dirpath) + ".tmp" if fileutil.Exist(tmpdirpath) { if err := os.RemoveAll(tmpdirpath); err != nil { return nil, err } } if err := fileutil.CreateDirAll(tmpdirpath); err != nil { return nil, err } p := path.Join(tmpdirpath, walName(0, 0)) f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode) if err != nil { return nil, err } if _, err := f.Seek(0, os.SEEK_END); err != nil { return nil, err } if err := fileutil.Preallocate(f.File, SegmentSizeBytes, true); err != nil { return nil, err } w := &WAL{ dir: dirpath, metadata: metadata, encoder: newEncoder(f, 0), } w.locks = append(w.locks, f) if err := w.saveCrc(0); err != nil { return nil, err } if err := w.encoder.encode(&walpb.Record{Type: metadataType, Data: metadata}); err != nil { return nil, err } if err := w.SaveSnapshot(walpb.Snapshot{}); err != nil { return nil, err } // rename of directory with locked files doesn't work on windows; close // the WAL to release the locks so the directory can be renamed w.Close() if err := os.Rename(tmpdirpath, dirpath); err != nil { return nil, err } // reopen and relock newWAL, oerr := Open(dirpath, walpb.Snapshot{}) if oerr != nil { return nil, oerr } if _, _, _, err := newWAL.ReadAll(); err != nil { newWAL.Close() return nil, err } return newWAL, nil }