func (p *puller) handleEmptyBlock(b bqBlock) {
f := b.file
of := p.openFiles[f.Name]
if b.last {
if of.err == nil {
of.file.Close()
}
}
if protocol.IsDeleted(f.Flags) {
if debug {
l.Debugf("pull: delete %q", f.Name)
}
os.Remove(of.temp)
// Ensure the file and the directory it is in is writeable so we can remove the file
dirName := filepath.Dir(of.filepath)
os.Chmod(of.filepath, 0666)
if dirName != p.repoCfg.Directory {
os.Chmod(dirName, 0777)
}
if p.versioner != nil {
if debug {
l.Debugln("pull: deleting with versioner")
}
if err := p.versioner.Archive(p.repoCfg.Directory, of.filepath); err == nil {
p.model.updateLocal(p.repoCfg.ID, f)
} else if debug {
l.Debugln("pull: error:", err)
}
} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
p.model.updateLocal(p.repoCfg.ID, f)
}
} else {
if debug {
l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name)
}
t := time.Unix(f.Modified, 0)
if os.Chtimes(of.temp, t, t) != nil {
delete(p.openFiles, f.Name)
return
}
if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil {
delete(p.openFiles, f.Name)
return
}
osutil.ShowFile(of.temp)
if osutil.Rename(of.temp, of.filepath) == nil {
p.model.updateLocal(p.repoCfg.ID, f)
}
}
delete(p.openFiles, f.Name)
}
func (p *puller) closeFile(f protocol.FileInfo) {
if debug {
l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name)
}
of := p.openFiles[f.Name]
err := of.file.Close()
if err != nil {
p.errors++
l.Infof("close: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
defer os.Remove(of.temp)
delete(p.openFiles, f.Name)
fd, err := os.Open(of.temp)
if err != nil {
p.errors++
l.Infof("open: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
return
}
hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize, f.Size())
fd.Close()
if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
if debug {
l.Debugf("pull: %q / %q: nblocks %d != %d", p.repoCfg.ID, f.Name, l0, l1)
}
return
}
for i := range hb {
if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
if debug {
l.Debugf("pull: %q / %q: block %d hash mismatch\n have: %x\n want: %x", p.repoCfg.ID, f.Name, i, hb[i].Hash, f.Blocks[i].Hash)
}
return
}
}
t := time.Unix(f.Modified, 0)
err = os.Chtimes(of.temp, t, t)
if err != nil {
l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
err = os.Chmod(of.temp, os.FileMode(f.Flags&0777))
if err != nil {
l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
}
osutil.ShowFile(of.temp)
if p.versioner != nil {
err := p.versioner.Archive(of.filepath)
if err != nil {
if debug {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
return
}
}
if debug {
l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath)
}
if err := osutil.Rename(of.temp, of.filepath); err == nil {
p.model.updateLocal(p.repoCfg.ID, f)
} else {
p.errors++
l.Infof("rename: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
}
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
f := b.file
// For directories, making sure they exist is enough.
// Deleted directories we mark as handled and delete later.
if protocol.IsDirectory(f.Flags) {
if !protocol.IsDeleted(f.Flags) {
path := filepath.Join(p.repoCfg.Directory, f.Name)
_, err := os.Stat(path)
if err != nil && os.IsNotExist(err) {
if debug {
l.Debugf("create dir: %v", f)
}
err = os.MkdirAll(path, os.FileMode(f.Flags&0777))
if err != nil {
p.errors++
l.Infof("mkdir: error: %q: %v", path, err)
}
}
} else if debug {
l.Debugf("ignore delete dir: %v", f)
}
p.model.updateLocal(p.repoCfg.ID, f)
return true
}
if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
// We are supposed to copy the entire file, and then fetch nothing.
// We don't actually need to make the copy.
if debug {
l.Debugln("taking shortcut:", f)
}
fp := filepath.Join(p.repoCfg.Directory, f.Name)
t := time.Unix(f.Modified, 0)
err := os.Chtimes(fp, t, t)
if err != nil {
l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
err = os.Chmod(fp, os.FileMode(f.Flags&0777))
if err != nil {
l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
}
events.Default.Log(events.ItemStarted, map[string]string{
"repo": p.repoCfg.ID,
"item": f.Name,
})
p.model.updateLocal(p.repoCfg.ID, f)
return true
}
of, ok := p.openFiles[f.Name]
of.done = b.last
if !ok {
if debug {
l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
}
events.Default.Log(events.ItemStarted, map[string]string{
"repo": p.repoCfg.ID,
"item": f.Name,
})
of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))
dirName := filepath.Dir(of.filepath)
info, err := os.Stat(dirName)
if err != nil {
err = os.MkdirAll(dirName, 0777)
if debug && err != nil {
l.Debugf("mkdir: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
} else {
// We need to make sure the directory is writeable so we can create files in it
if dirName != p.repoCfg.Directory {
err = os.Chmod(dirName, 0777)
if debug && err != nil {
l.Debugf("make writeable: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
}
// Change it back after creating the file, to minimize the time window with incorrect permissions
defer os.Chmod(dirName, info.Mode())
}
of.file, of.err = os.Create(of.temp)
if of.err != nil {
p.errors++
l.Infof("create: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
if !b.last {
p.openFiles[f.Name] = of
}
return true
}
osutil.HideFile(of.temp)
}
if of.err != nil {
// We have already failed this file.
if debug {
l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
}
if b.last {
delete(p.openFiles, f.Name)
}
return true
}
p.openFiles[f.Name] = of
switch {
case len(b.copy) > 0:
p.handleCopyBlock(b)
return true
case b.block.Size > 0:
return p.handleRequestBlock(b)
default:
p.handleEmptyBlock(b)
return true
}
}
// clean deletes orphaned temporary files and directories that should no
// longer exist.
func (p *puller) clean() {
var deleteDirs []string
var changed = 0
var walkFn = func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) {
os.Remove(path)
}
if !info.IsDir() {
return nil
}
rn, err := filepath.Rel(p.repoCfg.Directory, path)
if err != nil {
return nil
}
if rn == "." {
return nil
}
if filepath.Base(rn) == ".stversions" {
return filepath.SkipDir
}
cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
if cur.Name != rn {
// No matching dir in current list; weird
if debug {
l.Debugf("missing dir: %s; %v", rn, cur)
}
return nil
}
if protocol.IsDeleted(cur.Flags) {
if debug {
l.Debugf("queue delete dir: %v", cur)
}
// We queue the directories to delete since we walk the
// tree in depth first order and need to remove the
// directories in the opposite order.
deleteDirs = append(deleteDirs, path)
return nil
}
if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
if err != nil {
l.Warnf("Restoring folder flags: %q: %v", path, err)
} else {
changed++
if debug {
l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
}
}
}
return nil
}
for {
deleteDirs = nil
changed = 0
filepath.Walk(p.repoCfg.Directory, walkFn)
var deleted = 0
// Delete any queued directories
for i := len(deleteDirs) - 1; i >= 0; i-- {
dir := deleteDirs[i]
if debug {
l.Debugln("delete dir:", dir)
}
err := os.Remove(dir)
if err == nil {
deleted++
} else {
l.Warnln("Delete dir:", err)
}
}
if debug {
l.Debugf("changed %d, deleted %d dirs", changed, deleted)
}
if changed+deleted == 0 {
return
}
}
}
func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
if debug {
l.Debugln("error:", p, info, err)
}
return nil
}
rn, err := filepath.Rel(w.Dir, p)
if err != nil {
if debug {
l.Debugln("rel error:", p, err)
}
return nil
}
if rn == "." {
return nil
}
if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
// A temporary file
if debug {
l.Debugln("temporary:", rn)
}
return nil
}
if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
// An ignored file
if debug {
l.Debugln("ignored:", rn)
}
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) {
l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn)
return nil
}
if info.Mode().IsDir() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged {
return nil
}
}
var flags uint32 = protocol.FlagDirectory
if w.IgnorePerms {
flags |= protocol.FlagNoPermBits | 0777
} else {
flags |= uint32(info.Mode() & os.ModePerm)
}
f := protocol.FileInfo{
Name: rn,
Version: lamport.Default.Tick(0),
Flags: flags,
Modified: info.ModTime().Unix(),
}
if debug {
l.Debugln("dir:", f)
}
fchan <- f
return nil
}
if info.Mode().IsRegular() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
return nil
}
if debug {
l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
}
}
var flags = uint32(info.Mode() & os.ModePerm)
if w.IgnorePerms {
flags = protocol.FlagNoPermBits | 0666
}
fchan <- protocol.FileInfo{
Name: rn,
Version: lamport.Default.Tick(0),
Flags: flags,
Modified: info.ModTime().Unix(),
}
}
return nil
}
}