func (p *puller) handleEmptyBlock(b bqBlock) { f := b.file of := p.openFiles[f.Name] if b.last { if of.err == nil { of.file.Close() } } if protocol.IsDeleted(f.Flags) { if debug { l.Debugf("pull: delete %q", f.Name) } os.Remove(of.temp) // Ensure the file and the directory it is in is writeable so we can remove the file dirName := filepath.Dir(of.filepath) os.Chmod(of.filepath, 0666) if dirName != p.repoCfg.Directory { os.Chmod(dirName, 0777) } if p.versioner != nil { if debug { l.Debugln("pull: deleting with versioner") } if err := p.versioner.Archive(p.repoCfg.Directory, of.filepath); err == nil { p.model.updateLocal(p.repoCfg.ID, f) } else if debug { l.Debugln("pull: error:", err) } } else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) { p.model.updateLocal(p.repoCfg.ID, f) } } else { if debug { l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name) } t := time.Unix(f.Modified, 0) if os.Chtimes(of.temp, t, t) != nil { delete(p.openFiles, f.Name) return } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil { delete(p.openFiles, f.Name) return } osutil.ShowFile(of.temp) if osutil.Rename(of.temp, of.filepath) == nil { p.model.updateLocal(p.repoCfg.ID, f) } } delete(p.openFiles, f.Name) }
func (p *puller) closeFile(f protocol.FileInfo) { if debug { l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name) } of := p.openFiles[f.Name] err := of.file.Close() if err != nil { p.errors++ l.Infof("close: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } defer os.Remove(of.temp) delete(p.openFiles, f.Name) fd, err := os.Open(of.temp) if err != nil { p.errors++ l.Infof("open: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) return } hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize, f.Size()) fd.Close() if l0, l1 := len(hb), len(f.Blocks); l0 != l1 { if debug { l.Debugf("pull: %q / %q: nblocks %d != %d", p.repoCfg.ID, f.Name, l0, l1) } return } for i := range hb { if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 { if debug { l.Debugf("pull: %q / %q: block %d hash mismatch\n have: %x\n want: %x", p.repoCfg.ID, f.Name, i, hb[i].Hash, f.Blocks[i].Hash) } return } } t := time.Unix(f.Modified, 0) err = os.Chtimes(of.temp, t, t) if err != nil { l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) { err = os.Chmod(of.temp, os.FileMode(f.Flags&0777)) if err != nil { l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } osutil.ShowFile(of.temp) if p.versioner != nil { err := p.versioner.Archive(of.filepath) if err != nil { if debug { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } return } } if debug { l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath) } if err := osutil.Rename(of.temp, of.filepath); err == nil { p.model.updateLocal(p.repoCfg.ID, f) } else { p.errors++ l.Infof("rename: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } }
// handleBlock fulfills the block request by copying, ignoring or fetching // from the network. Returns true if the block was fully handled // synchronously, i.e. if the slot can be reused. func (p *puller) handleBlock(b bqBlock) bool { f := b.file // For directories, making sure they exist is enough. // Deleted directories we mark as handled and delete later. if protocol.IsDirectory(f.Flags) { if !protocol.IsDeleted(f.Flags) { path := filepath.Join(p.repoCfg.Directory, f.Name) _, err := os.Stat(path) if err != nil && os.IsNotExist(err) { if debug { l.Debugf("create dir: %v", f) } err = os.MkdirAll(path, os.FileMode(f.Flags&0777)) if err != nil { p.errors++ l.Infof("mkdir: error: %q: %v", path, err) } } } else if debug { l.Debugf("ignore delete dir: %v", f) } p.model.updateLocal(p.repoCfg.ID, f) return true } if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last { // We are supposed to copy the entire file, and then fetch nothing. // We don't actually need to make the copy. if debug { l.Debugln("taking shortcut:", f) } fp := filepath.Join(p.repoCfg.Directory, f.Name) t := time.Unix(f.Modified, 0) err := os.Chtimes(fp, t, t) if err != nil { l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) { err = os.Chmod(fp, os.FileMode(f.Flags&0777)) if err != nil { l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } events.Default.Log(events.ItemStarted, map[string]string{ "repo": p.repoCfg.ID, "item": f.Name, }) p.model.updateLocal(p.repoCfg.ID, f) return true } of, ok := p.openFiles[f.Name] of.done = b.last if !ok { if debug { l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name) } events.Default.Log(events.ItemStarted, map[string]string{ "repo": p.repoCfg.ID, "item": f.Name, }) of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name) of.filepath = filepath.Join(p.repoCfg.Directory, f.Name) of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name)) dirName := filepath.Dir(of.filepath) info, err := os.Stat(dirName) if err != nil { err = os.MkdirAll(dirName, 0777) if debug && err != nil { l.Debugf("mkdir: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } else { // We need to make sure the directory is writeable so we can create files in it if dirName != p.repoCfg.Directory { err = os.Chmod(dirName, 0777) if debug && err != nil { l.Debugf("make writeable: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } // Change it back after creating the file, to minimize the time window with incorrect permissions defer os.Chmod(dirName, info.Mode()) } of.file, of.err = os.Create(of.temp) if of.err != nil { p.errors++ l.Infof("create: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err) if !b.last { p.openFiles[f.Name] = of } return true } osutil.HideFile(of.temp) } if of.err != nil { // We have already failed this file. if debug { l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err) } if b.last { delete(p.openFiles, f.Name) } return true } p.openFiles[f.Name] = of switch { case len(b.copy) > 0: p.handleCopyBlock(b) return true case b.block.Size > 0: return p.handleRequestBlock(b) default: p.handleEmptyBlock(b) return true } }
// clean deletes orphaned temporary files and directories that should no // longer exist. func (p *puller) clean() { var deleteDirs []string var changed = 0 var walkFn = func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) { os.Remove(path) } if !info.IsDir() { return nil } rn, err := filepath.Rel(p.repoCfg.Directory, path) if err != nil { return nil } if rn == "." { return nil } if filepath.Base(rn) == ".stversions" { return filepath.SkipDir } cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn) if cur.Name != rn { // No matching dir in current list; weird if debug { l.Debugf("missing dir: %s; %v", rn, cur) } return nil } if protocol.IsDeleted(cur.Flags) { if debug { l.Debugf("queue delete dir: %v", cur) } // We queue the directories to delete since we walk the // tree in depth first order and need to remove the // directories in the opposite order. deleteDirs = append(deleteDirs, path) return nil } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) { err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm) if err != nil { l.Warnf("Restoring folder flags: %q: %v", path, err) } else { changed++ if debug { l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur) } } } return nil } for { deleteDirs = nil changed = 0 filepath.Walk(p.repoCfg.Directory, walkFn) var deleted = 0 // Delete any queued directories for i := len(deleteDirs) - 1; i >= 0; i-- { dir := deleteDirs[i] if debug { l.Debugln("delete dir:", dir) } err := os.Remove(dir) if err == nil { deleted++ } else { l.Warnln("Delete dir:", err) } } if debug { l.Debugf("changed %d, deleted %d dirs", changed, deleted) } if changed+deleted == 0 { return } } }
func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ign map[string][]string) filepath.WalkFunc { return func(p string, info os.FileInfo, err error) error { if err != nil { if debug { l.Debugln("error:", p, info, err) } return nil } rn, err := filepath.Rel(w.Dir, p) if err != nil { if debug { l.Debugln("rel error:", p, err) } return nil } if rn == "." { return nil } if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) { // A temporary file if debug { l.Debugln("temporary:", rn) } return nil } if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) { // An ignored file if debug { l.Debugln("ignored:", rn) } if info.IsDir() { return filepath.SkipDir } return nil } if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) { l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn) return nil } if info.Mode().IsDir() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode())) if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged { return nil } } var flags uint32 = protocol.FlagDirectory if w.IgnorePerms { flags |= protocol.FlagNoPermBits | 0777 } else { flags |= uint32(info.Mode() & os.ModePerm) } f := protocol.FileInfo{ Name: rn, Version: lamport.Default.Tick(0), Flags: flags, Modified: info.ModTime().Unix(), } if debug { l.Debugln("dir:", f) } fchan <- f return nil } if info.Mode().IsRegular() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode())) if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged { return nil } if debug { l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm) } } var flags = uint32(info.Mode() & os.ModePerm) if w.IgnorePerms { flags = protocol.FlagNoPermBits | 0666 } fchan <- protocol.FileInfo{ Name: rn, Version: lamport.Default.Tick(0), Flags: flags, Modified: info.ModTime().Unix(), } } return nil } }