func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
if !protocol.IsDeleted(f.Flags) {
files++
if !protocol.IsDirectory(f.Flags) {
bytes += f.Size()
} else {
bytes += zeroEntrySize
}
} else {
deleted++
bytes += zeroEntrySize
}
return
}
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
for _, f := range fs {
if !protocol.IsDeleted(f.Flags) {
files++
if !protocol.IsDirectory(f.Flags) {
bytes += f.Size
} else {
bytes += zeroEntrySize
}
} else {
deleted++
bytes += zeroEntrySize
}
}
return
}
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
f := b.file
// For directories, making sure they exist is enough.
// Deleted directories we mark as handled and delete later.
if protocol.IsDirectory(f.Flags) {
if !protocol.IsDeleted(f.Flags) {
path := filepath.Join(p.repoCfg.Directory, f.Name)
_, err := os.Stat(path)
if err != nil && os.IsNotExist(err) {
if debug {
l.Debugf("create dir: %v", f)
}
err = os.MkdirAll(path, 0777)
if err != nil {
l.Warnf("Create folder: %q: %v", path, err)
}
}
} else if debug {
l.Debugf("ignore delete dir: %v", f)
}
p.model.updateLocal(p.repoCfg.ID, f)
return true
}
if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
// We are supposed to copy the entire file, and then fetch nothing.
// We don't actually need to make the copy.
if debug {
l.Debugln("taking shortcut:", f)
}
fp := filepath.Join(p.repoCfg.Directory, f.Name)
t := time.Unix(f.Modified, 0)
err := os.Chtimes(fp, t, t)
if debug && err != nil {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
err = os.Chmod(fp, os.FileMode(f.Flags&0777))
if debug && err != nil {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
}
p.model.updateLocal(p.repoCfg.ID, f)
return true
}
of, ok := p.openFiles[f.Name]
of.done = b.last
if !ok {
if debug {
l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
}
of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))
dirName := filepath.Dir(of.filepath)
_, err := os.Stat(dirName)
if err != nil {
err = os.MkdirAll(dirName, 0777)
}
if err != nil {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
of.file, of.err = os.Create(of.temp)
if of.err != nil {
if debug {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
}
if !b.last {
p.openFiles[f.Name] = of
}
return true
}
osutil.HideFile(of.temp)
}
if of.err != nil {
// We have already failed this file.
if debug {
l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
}
if b.last {
delete(p.openFiles, f.Name)
}
return true
}
p.openFiles[f.Name] = of
switch {
case len(b.copy) > 0:
p.handleCopyBlock(b)
return true
case b.block.Size > 0:
return p.handleRequestBlock(b)
default:
p.handleEmptyBlock(b)
return true
}
}
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
if debug {
l.Debugln("error:", p, info, err)
}
return nil
}
rn, err := filepath.Rel(w.Dir, p)
if err != nil {
if debug {
l.Debugln("rel error:", p, err)
}
return nil
}
if rn == "." {
return nil
}
if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
// A temporary file
if debug {
l.Debugln("temporary:", rn)
}
return nil
}
if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
// An ignored file
if debug {
l.Debugln("ignored:", rn)
}
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.Mode().IsDir() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
if debug {
l.Debugln("unchanged:", cf)
}
*res = append(*res, cf)
} else {
var flags uint32 = protocol.FlagDirectory
if w.IgnorePerms {
flags |= protocol.FlagNoPermBits | 0777
} else {
flags |= uint32(info.Mode() & os.ModePerm)
}
f := File{
Name: rn,
Version: lamport.Default.Tick(0),
Flags: flags,
Modified: info.ModTime().Unix(),
}
if debug {
l.Debugln("dir:", cf, f)
}
*res = append(*res, f)
}
return nil
}
}
if info.Mode().IsRegular() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
if debug {
l.Debugln("unchanged:", cf)
}
*res = append(*res, cf)
return nil
}
if w.Suppressor != nil {
if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
cf.Suppressed = true
cf.Version++
if debug {
l.Debugln("suppressed:", cf)
}
*res = append(*res, cf)
return nil
} else if prev && !cur {
l.Infof("Changes to %q are no longer suppressed.", p)
}
}
if debug {
l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
}
}
fd, err := os.Open(p)
if err != nil {
if debug {
l.Debugln("open:", p, err)
}
return nil
}
defer fd.Close()
t0 := time.Now()
blocks, err := Blocks(fd, w.BlockSize)
if err != nil {
if debug {
l.Debugln("hash error:", rn, err)
}
return nil
}
if debug {
t1 := time.Now()
l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
}
var flags = uint32(info.Mode() & os.ModePerm)
if w.IgnorePerms {
flags = protocol.FlagNoPermBits | 0666
}
f := File{
Name: rn,
Version: lamport.Default.Tick(0),
Size: info.Size(),
Flags: flags,
Modified: info.ModTime().Unix(),
Blocks: blocks,
}
*res = append(*res, f)
}
return nil
}
}
// ConnectionStats returns a map with connection statistics for each connected node.
func (m *Model) ConnectionStats() map[string]ConnectionInfo {
type remoteAddrer interface {
RemoteAddr() net.Addr
}
m.pmut.RLock()
m.rmut.RLock()
var res = make(map[string]ConnectionInfo)
for node, conn := range m.protoConn {
ci := ConnectionInfo{
Statistics: conn.Statistics(),
ClientVersion: m.nodeVer[node],
}
if nc, ok := m.rawConn[node].(remoteAddrer); ok {
ci.Address = nc.RemoteAddr().String()
}
var tot int64
var have int64
for _, repo := range m.nodeRepos[node] {
m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
if !protocol.IsDeleted(f.Flags) {
var size int64
if protocol.IsDirectory(f.Flags) {
size = zeroEntrySize
} else {
size = f.Size()
}
tot += size
have += size
}
return true
})
m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
if !protocol.IsDeleted(f.Flags) {
var size int64
if protocol.IsDirectory(f.Flags) {
size = zeroEntrySize
} else {
size = f.Size()
}
have -= size
}
return true
})
}
ci.Completion = 100
if tot != 0 {
ci.Completion = int(100 * have / tot)
}
res[node.String()] = ci
}
m.rmut.RUnlock()
m.pmut.RUnlock()
in, out := protocol.TotalInOut()
res["total"] = ConnectionInfo{
Statistics: protocol.Statistics{
At: time.Now(),
InBytesTotal: in,
OutBytesTotal: out,
},
}
return res
}
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
f := b.file
// For directories, making sure they exist is enough.
// Deleted directories we mark as handled and delete later.
if protocol.IsDirectory(f.Flags) {
if !protocol.IsDeleted(f.Flags) {
path := filepath.Join(p.repoCfg.Directory, f.Name)
_, err := os.Stat(path)
if err != nil && os.IsNotExist(err) {
if debug {
l.Debugf("create dir: %v", f)
}
err = os.MkdirAll(path, 0777)
if err != nil {
l.Warnf("Create folder: %q: %v", path, err)
}
}
} else if debug {
l.Debugf("ignore delete dir: %v", f)
}
p.model.updateLocal(p.repoCfg.ID, f)
return true
}
of, ok := p.openFiles[f.Name]
of.done = b.last
if !ok {
if debug {
l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
}
of.availability = uint64(p.model.repoFiles[p.repoCfg.ID].Availability(f.Name))
of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))
dirName := filepath.Dir(of.filepath)
_, err := os.Stat(dirName)
if err != nil {
err = os.MkdirAll(dirName, 0777)
}
if err != nil {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
of.file, of.err = os.Create(of.temp)
if of.err != nil {
if debug {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
}
if !b.last {
p.openFiles[f.Name] = of
}
return true
}
osutil.HideFile(of.temp)
}
if of.err != nil {
// We have already failed this file.
if debug {
l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
}
if b.last {
delete(p.openFiles, f.Name)
}
return true
}
p.openFiles[f.Name] = of
switch {
case len(b.copy) > 0:
p.handleCopyBlock(b)
return true
case b.block.Size > 0:
return p.handleRequestBlock(b)
default:
p.handleEmptyBlock(b)
return true
}
}
