Пример #1
0
func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
	if !protocol.IsDeleted(f.Flags) {
		files++
		if !protocol.IsDirectory(f.Flags) {
			bytes += f.Size()
		} else {
			bytes += zeroEntrySize
		}
	} else {
		deleted++
		bytes += zeroEntrySize
	}
	return
}
Пример #2
0
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
	for _, f := range fs {
		if !protocol.IsDeleted(f.Flags) {
			files++
			if !protocol.IsDirectory(f.Flags) {
				bytes += f.Size
			} else {
				bytes += zeroEntrySize
			}
		} else {
			deleted++
			bytes += zeroEntrySize
		}
	}
	return
}
Пример #3
0
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
	f := b.file

	// For directories, making sure they exist is enough.
	// Deleted directories we mark as handled and delete later.
	if protocol.IsDirectory(f.Flags) {
		if !protocol.IsDeleted(f.Flags) {
			path := filepath.Join(p.repoCfg.Directory, f.Name)
			_, err := os.Stat(path)
			if err != nil && os.IsNotExist(err) {
				if debug {
					l.Debugf("create dir: %v", f)
				}
				err = os.MkdirAll(path, 0777)
				if err != nil {
					l.Warnf("Create folder: %q: %v", path, err)
				}
			}
		} else if debug {
			l.Debugf("ignore delete dir: %v", f)
		}
		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
		// We are supposed to copy the entire file, and then fetch nothing.
		// We don't actually need to make the copy.
		if debug {
			l.Debugln("taking shortcut:", f)
		}
		fp := filepath.Join(p.repoCfg.Directory, f.Name)
		t := time.Unix(f.Modified, 0)
		err := os.Chtimes(fp, t, t)
		if debug && err != nil {
			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
			err = os.Chmod(fp, os.FileMode(f.Flags&0777))
			if debug && err != nil {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
		}

		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	of, ok := p.openFiles[f.Name]
	of.done = b.last

	if !ok {
		if debug {
			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
		}

		of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))

		dirName := filepath.Dir(of.filepath)
		_, err := os.Stat(dirName)
		if err != nil {
			err = os.MkdirAll(dirName, 0777)
		}
		if err != nil {
			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}

		of.file, of.err = os.Create(of.temp)
		if of.err != nil {
			if debug {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
			}
			if !b.last {
				p.openFiles[f.Name] = of
			}
			return true
		}
		osutil.HideFile(of.temp)
	}

	if of.err != nil {
		// We have already failed this file.
		if debug {
			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
		}
		if b.last {
			delete(p.openFiles, f.Name)
		}

		return true
	}

	p.openFiles[f.Name] = of

	switch {
	case len(b.copy) > 0:
		p.handleCopyBlock(b)
		return true

	case b.block.Size > 0:
		return p.handleRequestBlock(b)

	default:
		p.handleEmptyBlock(b)
		return true
	}
}
Пример #4
0
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					var flags uint32 = protocol.FlagDirectory
					if w.IgnorePerms {
						flags |= protocol.FlagNoPermBits | 0777
					} else {
						flags |= uint32(info.Mode() & os.ModePerm)
					}
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    flags,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}
Пример #5
0
// ConnectionStats returns a map with connection statistics for each connected node.
func (m *Model) ConnectionStats() map[string]ConnectionInfo {
	type remoteAddrer interface {
		RemoteAddr() net.Addr
	}

	m.pmut.RLock()
	m.rmut.RLock()

	var res = make(map[string]ConnectionInfo)
	for node, conn := range m.protoConn {
		ci := ConnectionInfo{
			Statistics:    conn.Statistics(),
			ClientVersion: m.nodeVer[node],
		}
		if nc, ok := m.rawConn[node].(remoteAddrer); ok {
			ci.Address = nc.RemoteAddr().String()
		}

		var tot int64
		var have int64

		for _, repo := range m.nodeRepos[node] {
			m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
				if !protocol.IsDeleted(f.Flags) {
					var size int64
					if protocol.IsDirectory(f.Flags) {
						size = zeroEntrySize
					} else {
						size = f.Size()
					}
					tot += size
					have += size
				}
				return true
			})

			m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
				if !protocol.IsDeleted(f.Flags) {
					var size int64
					if protocol.IsDirectory(f.Flags) {
						size = zeroEntrySize
					} else {
						size = f.Size()
					}
					have -= size
				}
				return true
			})
		}

		ci.Completion = 100
		if tot != 0 {
			ci.Completion = int(100 * have / tot)
		}

		res[node.String()] = ci
	}

	m.rmut.RUnlock()
	m.pmut.RUnlock()

	in, out := protocol.TotalInOut()
	res["total"] = ConnectionInfo{
		Statistics: protocol.Statistics{
			At:            time.Now(),
			InBytesTotal:  in,
			OutBytesTotal: out,
		},
	}

	return res
}
Пример #6
0
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
	f := b.file

	// For directories, making sure they exist is enough.
	// Deleted directories we mark as handled and delete later.
	if protocol.IsDirectory(f.Flags) {
		if !protocol.IsDeleted(f.Flags) {
			path := filepath.Join(p.repoCfg.Directory, f.Name)
			_, err := os.Stat(path)
			if err != nil && os.IsNotExist(err) {
				if debug {
					l.Debugf("create dir: %v", f)
				}
				err = os.MkdirAll(path, 0777)
				if err != nil {
					l.Warnf("Create folder: %q: %v", path, err)
				}
			}
		} else if debug {
			l.Debugf("ignore delete dir: %v", f)
		}
		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	of, ok := p.openFiles[f.Name]
	of.done = b.last

	if !ok {
		if debug {
			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
		}

		of.availability = uint64(p.model.repoFiles[p.repoCfg.ID].Availability(f.Name))
		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))

		dirName := filepath.Dir(of.filepath)
		_, err := os.Stat(dirName)
		if err != nil {
			err = os.MkdirAll(dirName, 0777)
		}
		if err != nil {
			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}

		of.file, of.err = os.Create(of.temp)
		if of.err != nil {
			if debug {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
			}
			if !b.last {
				p.openFiles[f.Name] = of
			}
			return true
		}
		osutil.HideFile(of.temp)
	}

	if of.err != nil {
		// We have already failed this file.
		if debug {
			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
		}
		if b.last {
			delete(p.openFiles, f.Name)
		}

		return true
	}

	p.openFiles[f.Name] = of

	switch {
	case len(b.copy) > 0:
		p.handleCopyBlock(b)
		return true

	case b.block.Size > 0:
		return p.handleRequestBlock(b)

	default:
		p.handleEmptyBlock(b)
		return true
	}
}