func (p *puller) handleEmptyBlock(b bqBlock) {
	f := b.file
	of := p.openFiles[f.Name]

	if b.last {
		if of.err == nil {
			of.file.Close()
		}
	}

	if protocol.IsDeleted(f.Flags) {
		if debug {
			l.Debugf("pull: delete %q", f.Name)
		}
		os.Remove(of.temp)

		// Ensure the file and the directory it is in is writeable so we can remove the file
		dirName := filepath.Dir(of.filepath)
		os.Chmod(of.filepath, 0666)
		if dirName != p.repoCfg.Directory {
			os.Chmod(dirName, 0777)
		}
		if p.versioner != nil {
			if debug {
				l.Debugln("pull: deleting with versioner")
			}
			if err := p.versioner.Archive(p.repoCfg.Directory, of.filepath); err == nil {
				p.model.updateLocal(p.repoCfg.ID, f)
			} else if debug {
				l.Debugln("pull: error:", err)
			}
		} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
			p.model.updateLocal(p.repoCfg.ID, f)
		}
	} else {
		if debug {
			l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name)
		}
		t := time.Unix(f.Modified, 0)
		if os.Chtimes(of.temp, t, t) != nil {
			delete(p.openFiles, f.Name)
			return
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil {
			delete(p.openFiles, f.Name)
			return
		}
		osutil.ShowFile(of.temp)
		if osutil.Rename(of.temp, of.filepath) == nil {
			p.model.updateLocal(p.repoCfg.ID, f)
		}
	}
	delete(p.openFiles, f.Name)
}
Exemple #2
0
func (p *puller) closeFile(f protocol.FileInfo) {
	if debug {
		l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name)
	}

	of := p.openFiles[f.Name]
	err := of.file.Close()
	if err != nil {
		p.errors++
		l.Infof("close: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
	}
	defer os.Remove(of.temp)

	delete(p.openFiles, f.Name)

	fd, err := os.Open(of.temp)
	if err != nil {
		p.errors++
		l.Infof("open: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		return
	}
	hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize, f.Size())
	fd.Close()

	if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
		if debug {
			l.Debugf("pull: %q / %q: nblocks %d != %d", p.repoCfg.ID, f.Name, l0, l1)
		}
		return
	}

	for i := range hb {
		if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
			if debug {
				l.Debugf("pull: %q / %q: block %d hash mismatch\n  have: %x\n  want: %x", p.repoCfg.ID, f.Name, i, hb[i].Hash, f.Blocks[i].Hash)
			}
			return
		}
	}

	t := time.Unix(f.Modified, 0)
	err = os.Chtimes(of.temp, t, t)
	if err != nil {
		l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
	}
	if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
		err = os.Chmod(of.temp, os.FileMode(f.Flags&0777))
		if err != nil {
			l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}
	}

	osutil.ShowFile(of.temp)

	if p.versioner != nil {
		err := p.versioner.Archive(of.filepath)
		if err != nil {
			if debug {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
			return
		}
	}

	if debug {
		l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath)
	}
	if err := osutil.Rename(of.temp, of.filepath); err == nil {
		p.model.updateLocal(p.repoCfg.ID, f)
	} else {
		p.errors++
		l.Infof("rename: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
	}
}
Exemple #3
0
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
	f := b.file

	// For directories, making sure they exist is enough.
	// Deleted directories we mark as handled and delete later.
	if protocol.IsDirectory(f.Flags) {
		if !protocol.IsDeleted(f.Flags) {
			path := filepath.Join(p.repoCfg.Directory, f.Name)
			_, err := os.Stat(path)
			if err != nil && os.IsNotExist(err) {
				if debug {
					l.Debugf("create dir: %v", f)
				}
				err = os.MkdirAll(path, os.FileMode(f.Flags&0777))
				if err != nil {
					p.errors++
					l.Infof("mkdir: error: %q: %v", path, err)
				}
			}
		} else if debug {
			l.Debugf("ignore delete dir: %v", f)
		}
		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
		// We are supposed to copy the entire file, and then fetch nothing.
		// We don't actually need to make the copy.
		if debug {
			l.Debugln("taking shortcut:", f)
		}
		fp := filepath.Join(p.repoCfg.Directory, f.Name)
		t := time.Unix(f.Modified, 0)
		err := os.Chtimes(fp, t, t)
		if err != nil {
			l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
			err = os.Chmod(fp, os.FileMode(f.Flags&0777))
			if err != nil {
				l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
		}

		events.Default.Log(events.ItemStarted, map[string]string{
			"repo": p.repoCfg.ID,
			"item": f.Name,
		})

		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	of, ok := p.openFiles[f.Name]
	of.done = b.last

	if !ok {
		if debug {
			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
		}

		events.Default.Log(events.ItemStarted, map[string]string{
			"repo": p.repoCfg.ID,
			"item": f.Name,
		})

		of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))

		dirName := filepath.Dir(of.filepath)
		info, err := os.Stat(dirName)
		if err != nil {
			err = os.MkdirAll(dirName, 0777)
			if debug && err != nil {
				l.Debugf("mkdir: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
		} else {
			// We need to make sure the directory is writeable so we can create files in it
			if dirName != p.repoCfg.Directory {
				err = os.Chmod(dirName, 0777)
				if debug && err != nil {
					l.Debugf("make writeable: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
				}
			}
			// Change it back after creating the file, to minimize the time window with incorrect permissions
			defer os.Chmod(dirName, info.Mode())
		}

		of.file, of.err = os.Create(of.temp)
		if of.err != nil {
			p.errors++
			l.Infof("create: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
			if !b.last {
				p.openFiles[f.Name] = of
			}
			return true
		}
		osutil.HideFile(of.temp)
	}

	if of.err != nil {
		// We have already failed this file.
		if debug {
			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
		}
		if b.last {
			delete(p.openFiles, f.Name)
		}

		return true
	}

	p.openFiles[f.Name] = of

	switch {
	case len(b.copy) > 0:
		p.handleCopyBlock(b)
		return true

	case b.block.Size > 0:
		return p.handleRequestBlock(b)

	default:
		p.handleEmptyBlock(b)
		return true
	}
}
Exemple #4
0
// clean deletes orphaned temporary files and directories that should no
// longer exist.
func (p *puller) clean() {
	var deleteDirs []string
	var changed = 0

	var walkFn = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) {
			os.Remove(path)
		}

		if !info.IsDir() {
			return nil
		}

		rn, err := filepath.Rel(p.repoCfg.Directory, path)
		if err != nil {
			return nil
		}

		if rn == "." {
			return nil
		}

		if filepath.Base(rn) == ".stversions" {
			return filepath.SkipDir
		}

		cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
		if cur.Name != rn {
			// No matching dir in current list; weird
			if debug {
				l.Debugf("missing dir: %s; %v", rn, cur)
			}
			return nil
		}

		if protocol.IsDeleted(cur.Flags) {
			if debug {
				l.Debugf("queue delete dir: %v", cur)
			}

			// We queue the directories to delete since we walk the
			// tree in depth first order and need to remove the
			// directories in the opposite order.

			deleteDirs = append(deleteDirs, path)
			return nil
		}

		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
			err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
			if err != nil {
				l.Warnf("Restoring folder flags: %q: %v", path, err)
			} else {
				changed++
				if debug {
					l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
				}
			}
		}

		return nil
	}

	for {
		deleteDirs = nil
		changed = 0
		filepath.Walk(p.repoCfg.Directory, walkFn)

		var deleted = 0
		// Delete any queued directories
		for i := len(deleteDirs) - 1; i >= 0; i-- {
			dir := deleteDirs[i]
			if debug {
				l.Debugln("delete dir:", dir)
			}
			err := os.Remove(dir)
			if err == nil {
				deleted++
			} else {
				l.Warnln("Delete dir:", err)
			}
		}

		if debug {
			l.Debugf("changed %d, deleted %d dirs", changed, deleted)
		}

		if changed+deleted == 0 {
			return
		}
	}
}
func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) {
			l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn)
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged {
					return nil
				}
			}

			var flags uint32 = protocol.FlagDirectory
			if w.IgnorePerms {
				flags |= protocol.FlagNoPermBits | 0777
			} else {
				flags |= uint32(info.Mode() & os.ModePerm)
			}
			f := protocol.FileInfo{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
			}
			if debug {
				l.Debugln("dir:", f)
			}
			fchan <- f
			return nil
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					return nil
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}

			fchan <- protocol.FileInfo{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
			}
		}

		return nil
	}
}