// handleBlock fulfills the block request by copying, ignoring or fetching // from the network. Returns true if the block was fully handled // synchronously, i.e. if the slot can be reused. func (p *puller) handleBlock(b bqBlock) bool { f := b.file // For directories, making sure they exist is enough. // Deleted directories we mark as handled and delete later. if protocol.IsDirectory(f.Flags) { if !protocol.IsDeleted(f.Flags) { path := filepath.Join(p.repoCfg.Directory, f.Name) _, err := os.Stat(path) if err != nil && os.IsNotExist(err) { if debug { l.Debugf("create dir: %v", f) } err = os.MkdirAll(path, os.FileMode(f.Flags&0777)) if err != nil { p.errors++ l.Infof("mkdir: error: %q: %v", path, err) } } } else if debug { l.Debugf("ignore delete dir: %v", f) } p.model.updateLocal(p.repoCfg.ID, f) return true } if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last { // We are supposed to copy the entire file, and then fetch nothing. // We don't actually need to make the copy. if debug { l.Debugln("taking shortcut:", f) } fp := filepath.Join(p.repoCfg.Directory, f.Name) t := time.Unix(f.Modified, 0) err := os.Chtimes(fp, t, t) if err != nil { l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) { err = os.Chmod(fp, os.FileMode(f.Flags&0777)) if err != nil { l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } events.Default.Log(events.ItemStarted, map[string]string{ "repo": p.repoCfg.ID, "item": f.Name, }) p.model.updateLocal(p.repoCfg.ID, f) return true } of, ok := p.openFiles[f.Name] of.done = b.last if !ok { if debug { l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name) } events.Default.Log(events.ItemStarted, map[string]string{ "repo": p.repoCfg.ID, "item": f.Name, }) of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name) of.filepath = filepath.Join(p.repoCfg.Directory, f.Name) of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name)) dirName := filepath.Dir(of.filepath) info, err := os.Stat(dirName) if err != nil { err = os.MkdirAll(dirName, 0777) if debug && err != nil { l.Debugf("mkdir: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } else { // We need to make sure the directory is writeable so we can create files in it if dirName != p.repoCfg.Directory { err = os.Chmod(dirName, 0777) if debug && err != nil { l.Debugf("make writeable: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } // Change it back after creating the file, to minimize the time window with incorrect permissions defer os.Chmod(dirName, info.Mode()) } of.file, of.err = os.Create(of.temp) if of.err != nil { p.errors++ l.Infof("create: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err) if !b.last { p.openFiles[f.Name] = of } return true } osutil.HideFile(of.temp) } if of.err != nil { // We have already failed this file. if debug { l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err) } if b.last { delete(p.openFiles, f.Name) } return true } p.openFiles[f.Name] = of switch { case len(b.copy) > 0: p.handleCopyBlock(b) return true case b.block.Size > 0: return p.handleRequestBlock(b) default: p.handleEmptyBlock(b) return true } }
// Move away the named file to a version archive. If this function returns // nil, the named file does not exist any more (has been archived). func (v Simple) Archive(repoPath, filePath string) error { _, err := os.Stat(filePath) if err != nil && os.IsNotExist(err) { if debug { l.Debugln("not archiving nonexistent file", filePath) } return nil } versionsDir := filepath.Join(repoPath, ".stversions") _, err = os.Stat(versionsDir) if err != nil { if os.IsNotExist(err) { if debug { l.Debugln("creating versions dir", versionsDir) } os.MkdirAll(versionsDir, 0755) osutil.HideFile(versionsDir) } else { return err } } if debug { l.Debugln("archiving", filePath) } file := filepath.Base(filePath) inRepoPath, err := filepath.Rel(repoPath, filepath.Dir(filePath)) if err != nil { return err } dir := filepath.Join(versionsDir, inRepoPath) err = os.MkdirAll(dir, 0755) if err != nil && !os.IsExist(err) { return err } ver := file + "~" + time.Now().Format("20060102-150405") dst := filepath.Join(dir, ver) if debug { l.Debugln("moving to", dst) } err = osutil.Rename(filePath, dst) if err != nil { return err } versions, err := filepath.Glob(filepath.Join(dir, file+"~*")) if err != nil { l.Warnln(err) return nil } if len(versions) > v.keep { sort.Strings(versions) for _, toRemove := range versions[:len(versions)-v.keep] { if debug { l.Debugln("cleaning out", toRemove) } err = os.Remove(toRemove) if err != nil { l.Warnln(err) } } } return nil }
func (v Staggered) clean() { if debug { l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath) } v.mutex.Lock() defer v.mutex.Unlock() if debug { l.Debugln("Versioner clean: Cleaning", v.versionsPath) } _, err := os.Stat(v.versionsPath) if err != nil { if os.IsNotExist(err) { if debug { l.Debugln("creating versions dir", v.versionsPath) } os.MkdirAll(v.versionsPath, 0755) osutil.HideFile(v.versionsPath) } else { l.Warnln("Versioner: can't create versions dir", err) } } versionsPerFile := make(map[string][]string) filesPerDir := make(map[string]int) err = filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error { switch mode := f.Mode(); { case mode.IsDir(): filesPerDir[path] = 0 case mode.IsRegular(): extension := filepath.Ext(path) dir := filepath.Dir(path) name := path[:len(path)-len(extension)] filesPerDir[dir]++ versionsPerFile[name] = append(versionsPerFile[name], path) } return nil }) if err != nil { l.Warnln("Versioner: error scanning versions dir", err) return } for _, versionList := range versionsPerFile { // List from filepath.Walk is sorted v.expire(versionList) } for path, numFiles := range filesPerDir { if path == v.versionsPath { if debug { l.Debugln("Cleaner: versions dir is empty, don't delete", path) } continue } if numFiles > 0 { continue } if debug { l.Debugln("Cleaner: deleting empty directory", path) } err = os.Remove(path) if err != nil { l.Warnln("Versioner: can't remove directory", path, err) } } if debug { l.Debugln("Cleaner: Finished cleaning", v.versionsPath) } }
// Move away the named file to a version archive. If this function returns // nil, the named file does not exist any more (has been archived). func (v Staggered) Archive(filePath string) error { if debug { l.Debugln("Waiting for lock on ", v.versionsPath) } v.mutex.Lock() defer v.mutex.Unlock() _, err := os.Stat(filePath) if err != nil { if os.IsNotExist(err) { if debug { l.Debugln("not archiving nonexistent file", filePath) } return nil } else { return err } } _, err = os.Stat(v.versionsPath) if err != nil { if os.IsNotExist(err) { if debug { l.Debugln("creating versions dir", v.versionsPath) } os.MkdirAll(v.versionsPath, 0755) osutil.HideFile(v.versionsPath) } else { return err } } if debug { l.Debugln("archiving", filePath) } file := filepath.Base(filePath) inRepoPath, err := filepath.Rel(v.repoPath, filepath.Dir(filePath)) if err != nil { return err } dir := filepath.Join(v.versionsPath, inRepoPath) err = os.MkdirAll(dir, 0755) if err != nil && !os.IsExist(err) { return err } ver := file + ".v" + fmt.Sprintf("%010d", time.Now().Unix()) dst := filepath.Join(dir, ver) if debug { l.Debugln("moving to", dst) } err = osutil.Rename(filePath, dst) if err != nil { return err } versions, err := filepath.Glob(filepath.Join(dir, file+".v[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]")) if err != nil { l.Warnln("Versioner: error finding versions for", file, err) return nil } sort.Strings(versions) v.expire(versions) return nil }
// Move away the named file to a version archive. If this function returns // nil, the named file does not exist any more (has been archived). func (v Staggered) Archive(filePath string) error { if debug { l.Debugln(logPrefix, "Waiting for lock on ", v.versionsPath) } v.mutex.Lock() defer v.mutex.Unlock() fileInfo, err := os.Stat(filePath) if err != nil { if os.IsNotExist(err) { if debug { l.Debugln(logPrefix, "not archiving nonexistent file", filePath) } return nil } else { return err } } _, err = os.Stat(v.versionsPath) if err != nil { if os.IsNotExist(err) { if debug { l.Debugln(logPrefix, "creating versions dir", v.versionsPath) } os.MkdirAll(v.versionsPath, 0755) osutil.HideFile(v.versionsPath) } else { return err } } if debug { l.Debugln(logPrefix, "archiving", filePath) } file := filepath.Base(filePath) inRepoPath, err := filepath.Rel(v.repoPath, filepath.Dir(filePath)) if err != nil { return err } dir := filepath.Join(v.versionsPath, inRepoPath) err = os.MkdirAll(dir, 0755) if err != nil && !os.IsExist(err) { return err } ver := file + "~" + fileInfo.ModTime().Format(TimeLayout) dst := filepath.Join(dir, ver) if debug { l.Debugln(logPrefix, "moving to", dst) } err = osutil.Rename(filePath, dst) if err != nil { return err } versions, err := filepath.Glob(filepath.Join(dir, file+"~[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]")) if err != nil { l.Warnln(logPrefix, "Versioner: error finding versions for", file, err) return nil } sort.Strings(versions) v.expire(versions) return nil }