func (p *puller) handleEmptyBlock(b bqBlock) { f := b.file of := p.openFiles[f.Name] if b.last { if of.err == nil { of.file.Close() } } if protocol.IsDeleted(f.Flags) { if debug { l.Debugf("pull: delete %q", f.Name) } os.Remove(of.temp) os.Chmod(of.filepath, 0666) if p.versioner != nil { if debug { l.Debugln("pull: deleting with versioner") } if err := p.versioner.Archive(p.repoCfg.Directory, of.filepath); err == nil { p.model.updateLocal(p.repoCfg.ID, f) } else if debug { l.Debugln("pull: error:", err) } } else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) { p.model.updateLocal(p.repoCfg.ID, f) } } else { if debug { l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name) } t := time.Unix(f.Modified, 0) if os.Chtimes(of.temp, t, t) != nil { delete(p.openFiles, f.Name) return } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil { delete(p.openFiles, f.Name) return } osutil.ShowFile(of.temp) if osutil.Rename(of.temp, of.filepath) == nil { p.model.updateLocal(p.repoCfg.ID, f) } } delete(p.openFiles, f.Name) }
func (p *puller) closeFile(f scanner.File) { if debug { l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name) } of := p.openFiles[f.Name] of.file.Close() defer os.Remove(of.temp) delete(p.openFiles, f.Name) fd, err := os.Open(of.temp) if err != nil { if debug { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } return } hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize) fd.Close() if l0, l1 := len(hb), len(f.Blocks); l0 != l1 { if debug { l.Debugf("pull: %q / %q: nblocks %d != %d", p.repoCfg.ID, f.Name, l0, l1) } return } for i := range hb { if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 { l.Debugf("pull: %q / %q: block %d hash mismatch", p.repoCfg.ID, f.Name, i) return } } t := time.Unix(f.Modified, 0) err = os.Chtimes(of.temp, t, t) if debug && err != nil { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) { err = os.Chmod(of.temp, os.FileMode(f.Flags&0777)) if debug && err != nil { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } osutil.ShowFile(of.temp) if p.versioner != nil { err := p.versioner.Archive(of.filepath) if err != nil { if debug { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } return } } if debug { l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath) } if err := osutil.Rename(of.temp, of.filepath); err == nil { p.model.updateLocal(p.repoCfg.ID, f) } else { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } }
// handleBlock fulfills the block request by copying, ignoring or fetching // from the network. Returns true if the block was fully handled // synchronously, i.e. if the slot can be reused. func (p *puller) handleBlock(b bqBlock) bool { f := b.file // For directories, making sure they exist is enough. // Deleted directories we mark as handled and delete later. if protocol.IsDirectory(f.Flags) { if !protocol.IsDeleted(f.Flags) { path := filepath.Join(p.repoCfg.Directory, f.Name) _, err := os.Stat(path) if err != nil && os.IsNotExist(err) { if debug { l.Debugf("create dir: %v", f) } err = os.MkdirAll(path, 0777) if err != nil { l.Warnf("Create folder: %q: %v", path, err) } } } else if debug { l.Debugf("ignore delete dir: %v", f) } p.model.updateLocal(p.repoCfg.ID, f) return true } if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last { // We are supposed to copy the entire file, and then fetch nothing. // We don't actually need to make the copy. if debug { l.Debugln("taking shortcut:", f) } fp := filepath.Join(p.repoCfg.Directory, f.Name) t := time.Unix(f.Modified, 0) err := os.Chtimes(fp, t, t) if debug && err != nil { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) { err = os.Chmod(fp, os.FileMode(f.Flags&0777)) if debug && err != nil { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } } p.model.updateLocal(p.repoCfg.ID, f) return true } of, ok := p.openFiles[f.Name] of.done = b.last if !ok { if debug { l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name) } of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name) of.filepath = filepath.Join(p.repoCfg.Directory, f.Name) of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name)) dirName := filepath.Dir(of.filepath) _, err := os.Stat(dirName) if err != nil { err = os.MkdirAll(dirName, 0777) } if err != nil { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err) } of.file, of.err = os.Create(of.temp) if of.err != nil { if debug { l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err) } if !b.last { p.openFiles[f.Name] = of } return true } osutil.HideFile(of.temp) } if of.err != nil { // We have already failed this file. if debug { l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err) } if b.last { delete(p.openFiles, f.Name) } return true } p.openFiles[f.Name] = of switch { case len(b.copy) > 0: p.handleCopyBlock(b) return true case b.block.Size > 0: return p.handleRequestBlock(b) default: p.handleEmptyBlock(b) return true } }
func (p *puller) fixupDirectories() { var deleteDirs []string var changed = 0 var walkFn = func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { return nil } rn, err := filepath.Rel(p.repoCfg.Directory, path) if err != nil { return nil } if rn == "." { return nil } if filepath.Base(rn) == ".stversions" { return filepath.SkipDir } cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn) if cur.Name != rn { // No matching dir in current list; weird if debug { l.Debugf("missing dir: %s; %v", rn, cur) } return nil } if protocol.IsDeleted(cur.Flags) { if debug { l.Debugf("queue delete dir: %v", cur) } // We queue the directories to delete since we walk the // tree in depth first order and need to remove the // directories in the opposite order. deleteDirs = append(deleteDirs, path) return nil } if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) { err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm) if err != nil { l.Warnf("Restoring folder flags: %q: %v", path, err) } else { changed++ if debug { l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur) } } } if cur.Modified != info.ModTime().Unix() { t := time.Unix(cur.Modified, 0) err := os.Chtimes(path, t, t) if err != nil { if runtime.GOOS != "windows" { // https://code.google.com/p/go/issues/detail?id=8090 l.Warnf("Restoring folder modtime: %q: %v", path, err) } } else { changed++ if debug { l.Debugf("restored dir modtime: %d -> %v", info.ModTime().Unix(), cur) } } } return nil } for { deleteDirs = nil changed = 0 filepath.Walk(p.repoCfg.Directory, walkFn) var deleted = 0 // Delete any queued directories for i := len(deleteDirs) - 1; i >= 0; i-- { dir := deleteDirs[i] if debug { l.Debugln("delete dir:", dir) } err := os.Remove(dir) if err == nil { deleted++ } else if p.versioner == nil { // Failures are expected in the presence of versioning l.Warnln(err) } } if debug { l.Debugf("changed %d, deleted %d dirs", changed, deleted) } if changed+deleted == 0 { return } } }
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc { return func(p string, info os.FileInfo, err error) error { if err != nil { if debug { l.Debugln("error:", p, info, err) } return nil } rn, err := filepath.Rel(w.Dir, p) if err != nil { if debug { l.Debugln("rel error:", p, err) } return nil } if rn == "." { return nil } if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) { // A temporary file if debug { l.Debugln("temporary:", rn) } return nil } if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) { // An ignored file if debug { l.Debugln("ignored:", rn) } if info.IsDir() { return filepath.SkipDir } return nil } if info.Mode().IsDir() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode())) if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged { if debug { l.Debugln("unchanged:", cf) } *res = append(*res, cf) } else { var flags uint32 = protocol.FlagDirectory if w.IgnorePerms { flags |= protocol.FlagNoPermBits | 0777 } else { flags |= uint32(info.Mode() & os.ModePerm) } f := File{ Name: rn, Version: lamport.Default.Tick(0), Flags: flags, Modified: info.ModTime().Unix(), } if debug { l.Debugln("dir:", cf, f) } *res = append(*res, f) } return nil } } if info.Mode().IsRegular() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode())) if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged { if debug { l.Debugln("unchanged:", cf) } *res = append(*res, cf) return nil } if w.Suppressor != nil { if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev { l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p) cf.Suppressed = true cf.Version++ if debug { l.Debugln("suppressed:", cf) } *res = append(*res, cf) return nil } else if prev && !cur { l.Infof("Changes to %q are no longer suppressed.", p) } } if debug { l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm) } } fd, err := os.Open(p) if err != nil { if debug { l.Debugln("open:", p, err) } return nil } defer fd.Close() t0 := time.Now() blocks, err := Blocks(fd, w.BlockSize) if err != nil { if debug { l.Debugln("hash error:", rn, err) } return nil } if debug { t1 := time.Now() l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s") } var flags = uint32(info.Mode() & os.ModePerm) if w.IgnorePerms { flags = protocol.FlagNoPermBits | 0666 } f := File{ Name: rn, Version: lamport.Default.Tick(0), Size: info.Size(), Flags: flags, Modified: info.ModTime().Unix(), Blocks: blocks, } *res = append(*res, f) } return nil } }