Example #1
0
// IndexUpdate is called for incremental updates to connected nodes' indexes.
// Implements the protocol.Model interface.
func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo) {
	if debug {
		l.Debugf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
	}

	var files = make([]scanner.File, len(fs))
	for i := range fs {
		f := fs[i]
		lamport.Default.Tick(f.Version)
		if debug {
			var flagComment string
			if protocol.IsDeleted(f.Flags) {
				flagComment = " (deleted)"
			}
			l.Debugf("IDXUP(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
		}
		files[i] = fileFromFileInfo(f)
	}

	id := m.cm.Get(nodeID)
	m.rmut.RLock()
	if r, ok := m.repoFiles[repo]; ok {
		r.Update(id, files)
	} else {
		l.Warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
	}
	m.rmut.RUnlock()
}
Example #2
0
func (m *Model) ScanRepo(repo string) error {
	m.rmut.RLock()
	fs := m.repoFiles[repo]
	dir := m.repoCfgs[repo].Directory

	w := &scanner.Walker{
		Dir:          dir,
		IgnoreFile:   ".stignore",
		BlockSize:    scanner.StandardBlockSize,
		TempNamer:    defTempNamer,
		Suppressor:   m.suppressor[repo],
		CurrentFiler: cFiler{m, repo},
		IgnorePerms:  m.repoCfgs[repo].IgnorePerms,
	}
	m.rmut.RUnlock()

	m.setState(repo, RepoScanning)
	fchan, _, err := w.Walk()

	if err != nil {
		return err
	}
	batchSize := 100
	batch := make([]protocol.FileInfo, 0, 00)
	for f := range fchan {
		if len(batch) == batchSize {
			fs.Update(protocol.LocalNodeID, batch)
			batch = batch[:0]
		}
		batch = append(batch, f)
	}
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	batch = batch[:0]
	fs.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
		if !protocol.IsDeleted(f.Flags) {
			if len(batch) == batchSize {
				fs.Update(protocol.LocalNodeID, batch)
				batch = batch[:0]
			}
			if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
				// File has been deleted
				f.Blocks = nil
				f.Flags |= protocol.FlagDeleted
				f.Version = lamport.Default.Tick(f.Version)
				f.LocalVersion = 0
				batch = append(batch, f)
			}
		}
		return true
	})
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	m.setState(repo, RepoIdle)
	return nil
}
Example #3
0
// IndexUpdate is called for incremental updates to connected nodes' indexes.
// Implements the protocol.Model interface.
func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.FileInfo) {
	if debug {
		l.Debugf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
	}

	if !m.repoSharedWith(repo, nodeID) {
		l.Warnf("Unexpected repository ID %q sent from node %q; ensure that the repository exists and that this node is selected under \"Share With\" in the repository configuration.", repo, nodeID)
		return
	}

	var files = make([]scanner.File, len(fs))
	for i := range fs {
		f := fs[i]
		lamport.Default.Tick(f.Version)
		if debug {
			var flagComment string
			if protocol.IsDeleted(f.Flags) {
				flagComment = " (deleted)"
			}
			l.Debugf("IDXUP(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
		}
		files[i] = fileFromFileInfo(f)
	}

	m.rmut.RLock()
	if r, ok := m.repoFiles[repo]; ok {
		r.Update(nodeID, files)
	} else {
		l.Fatalf("IndexUpdate for nonexistant repo %q", repo)
	}
	m.rmut.RUnlock()
}
Example #4
0
func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
	if !protocol.IsDeleted(f.Flags) {
		files++
		if !protocol.IsDirectory(f.Flags) {
			bytes += f.Size()
		} else {
			bytes += zeroEntrySize
		}
	} else {
		deleted++
		bytes += zeroEntrySize
	}
	return
}
Example #5
0
func (m *Set) Need(id uint) []scanner.File {
	if debug {
		l.Debugf("Need(%d)", id)
	}
	m.Lock()
	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
	rkID := m.remoteKey[id]
	for gk, gf := range m.files {
		if !gf.Global || gf.File.Suppressed {
			continue
		}

		if rk, ok := rkID[gk.Name]; gk.newerThan(rk) {
			if protocol.IsDeleted(gf.File.Flags) && (!ok || protocol.IsDeleted(m.files[rk].File.Flags)) {
				// We don't need to delete files we don't have or that are already deleted
				continue
			}

			fs = append(fs, gf.File)
		}
	}
	m.Unlock()
	return fs
}
Example #6
0
func (p *puller) handleEmptyBlock(b bqBlock) {
	f := b.file
	of := p.openFiles[f.Name]

	if b.last {
		if of.err == nil {
			of.file.Close()
		}
	}

	if protocol.IsDeleted(f.Flags) {
		if debug {
			l.Debugf("pull: delete %q", f.Name)
		}
		os.Remove(of.temp)
		os.Chmod(of.filepath, 0666)
		if p.versioner != nil {
			if debug {
				l.Debugln("pull: deleting with versioner")
			}
			if err := p.versioner.Archive(p.repoCfg.Directory, of.filepath); err == nil {
				p.model.updateLocal(p.repoCfg.ID, f)
			} else if debug {
				l.Debugln("pull: error:", err)
			}
		} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
			p.model.updateLocal(p.repoCfg.ID, f)
		}
	} else {
		if debug {
			l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name)
		}
		t := time.Unix(f.Modified, 0)
		if os.Chtimes(of.temp, t, t) != nil {
			delete(p.openFiles, f.Name)
			return
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil {
			delete(p.openFiles, f.Name)
			return
		}
		osutil.ShowFile(of.temp)
		if osutil.Rename(of.temp, of.filepath) == nil {
			p.model.updateLocal(p.repoCfg.ID, f)
		}
	}
	delete(p.openFiles, f.Name)
}
Example #7
0
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
	for _, f := range fs {
		if !protocol.IsDeleted(f.Flags) {
			files++
			if !protocol.IsDirectory(f.Flags) {
				bytes += f.Size
			} else {
				bytes += zeroEntrySize
			}
		} else {
			deleted++
			bytes += zeroEntrySize
		}
	}
	return
}
Example #8
0
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64, size int) ([]byte, error) {
	// Verify that the requested file exists in the local model.
	m.rmut.RLock()
	r, ok := m.repoFiles[repo]
	m.rmut.RUnlock()

	if !ok {
		l.Warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
		return nil, ErrNoSuchFile
	}

	lf := r.Get(protocol.LocalNodeID, name)
	if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
		if debug {
			l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf)
		}
		return nil, ErrInvalid
	}

	if offset > lf.Size() {
		if debug {
			l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
		}
		return nil, ErrNoSuchFile
	}

	if debug && nodeID != protocol.LocalNodeID {
		l.Debugf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
	}
	m.rmut.RLock()
	fn := filepath.Join(m.repoCfgs[repo].Directory, name)
	m.rmut.RUnlock()
	fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	buf := make([]byte, size)
	_, err = fd.ReadAt(buf, offset)
	if err != nil {
		return nil, err
	}

	return buf, nil
}
Example #9
0
func (m *Set) equals(id uint, fs []scanner.File) bool {
	curWithoutDeleted := make(map[string]key)
	for _, k := range m.remoteKey[id] {
		f := m.files[k].File
		if !protocol.IsDeleted(f.Flags) {
			curWithoutDeleted[f.Name] = k
		}
	}
	if len(curWithoutDeleted) != len(fs) {
		return false
	}
	for _, f := range fs {
		if curWithoutDeleted[f.Name] != keyFor(f) {
			return false
		}
	}
	return true
}
Example #10
0
// protocolIndex returns the current local index in protocol data types.
func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
	var index []protocol.FileInfo

	fs := m.repoFiles[repo].Have(cid.LocalID)

	for _, f := range fs {
		mf := fileInfoFromFile(f)
		if debug {
			var flagComment string
			if protocol.IsDeleted(mf.Flags) {
				flagComment = " (deleted)"
			}
			l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
		}
		index = append(index, mf)
	}

	return index
}
Example #11
0
func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
	if debug {
		l.Debugf("ReplaceWithDelete(%d, [%d])", id, len(fs))
	}
	if id > 63 {
		panic("Connection ID must be in the range 0 - 63 inclusive")
	}

	m.Lock()
	if len(fs) == 0 || !m.equals(id, fs) {
		m.changes[id]++

		var nf = make(map[string]key, len(fs))
		for _, f := range fs {
			nf[f.Name] = keyFor(f)
		}

		// For previously existing files not in the list, add them to the list
		// with the relevant delete flags etc set. Previously existing files
		// with the delete bit already set are not modified.

		for _, ck := range m.remoteKey[cid.LocalID] {
			if _, ok := nf[ck.Name]; !ok {
				cf := m.files[ck].File
				if !protocol.IsDeleted(cf.Flags) {
					cf.Flags |= protocol.FlagDeleted
					cf.Blocks = nil
					cf.Size = 0
					cf.Version = lamport.Default.Tick(cf.Version)
				}
				fs = append(fs, cf)
				if debug {
					l.Debugln("deleted:", ck.Name)
				}
			}
		}

		m.replace(id, fs)
	}
	m.Unlock()
}
Example #12
0
func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
		var f protocol.FileInfo
		err := f.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if !protocol.IsDeleted(f.Flags) {
			if debug {
				l.Debugf("mark deleted; repo=%q node=%x name=%q", repo, node, name)
			}
			f.Blocks = nil
			f.Version = lamport.Default.Tick(f.Version)
			f.Flags |= protocol.FlagDeleted
			batch.Put(dbi.Key(), f.MarshalXDR())
			ldbUpdateGlobal(db, batch, repo, node, nodeKeyName(dbi.Key()), f.Version)
			return true
		}
		return false
	})
}
Example #13
0
// protocolIndex returns the current local index in protocol data types.
func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
	var index []protocol.FileInfo

	var fs []scanner.File
	m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f scanner.File) bool {
		fs = append(fs, f)
		return true
	})

	for _, f := range fs {
		mf := fileInfoFromFile(f)
		if debug {
			var flagComment string
			if protocol.IsDeleted(mf.Flags) {
				flagComment = " (deleted)"
			}
			l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
		}
		index = append(index, mf)
	}

	return index
}
Example #14
0
func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
	start := globalKey(repo, nil)
	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	for dbi.Next() {
		var vl versionList
		err := vl.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if len(vl.versions) == 0 {
			l.Debugln(dbi.Key())
			panic("no versions?")
		}

		have := false // If we have the file, any version
		need := false // If we have a lower version of the file
		var haveVersion uint64
		for _, v := range vl.versions {
			if bytes.Compare(v.node, node) == 0 {
				have = true
				haveVersion = v.version
				need = v.version < vl.versions[0].version
				break
			}
		}

		if need || !have {
			name := globalKeyName(dbi.Key())
			if debug {
				l.Debugf("need repo=%q node=%x name=%q need=%v have=%v haveV=%d globalV=%d", repo, node, name, need, have, haveVersion, vl.versions[0].version)
			}
			fk := nodeKey(repo, vl.versions[0].node, name)
			bs, err := snap.Get(fk, nil)
			if err != nil {
				panic(err)
			}

			var gf protocol.FileInfo
			err = gf.UnmarshalXDR(bs)
			if err != nil {
				panic(err)
			}

			if protocol.IsDeleted(gf.Flags) && !have {
				// We don't need deleted files that we don't have
				continue
			}

			if cont := fn(gf); !cont {
				return
			}
		}
	}
}
Example #15
0
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
	f := b.file

	// For directories, making sure they exist is enough.
	// Deleted directories we mark as handled and delete later.
	if protocol.IsDirectory(f.Flags) {
		if !protocol.IsDeleted(f.Flags) {
			path := filepath.Join(p.repoCfg.Directory, f.Name)
			_, err := os.Stat(path)
			if err != nil && os.IsNotExist(err) {
				if debug {
					l.Debugf("create dir: %v", f)
				}
				err = os.MkdirAll(path, 0777)
				if err != nil {
					l.Warnf("Create folder: %q: %v", path, err)
				}
			}
		} else if debug {
			l.Debugf("ignore delete dir: %v", f)
		}
		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
		// We are supposed to copy the entire file, and then fetch nothing.
		// We don't actually need to make the copy.
		if debug {
			l.Debugln("taking shortcut:", f)
		}
		fp := filepath.Join(p.repoCfg.Directory, f.Name)
		t := time.Unix(f.Modified, 0)
		err := os.Chtimes(fp, t, t)
		if debug && err != nil {
			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
			err = os.Chmod(fp, os.FileMode(f.Flags&0777))
			if debug && err != nil {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
		}

		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	of, ok := p.openFiles[f.Name]
	of.done = b.last

	if !ok {
		if debug {
			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
		}

		of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))

		dirName := filepath.Dir(of.filepath)
		_, err := os.Stat(dirName)
		if err != nil {
			err = os.MkdirAll(dirName, 0777)
		}
		if err != nil {
			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}

		of.file, of.err = os.Create(of.temp)
		if of.err != nil {
			if debug {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
			}
			if !b.last {
				p.openFiles[f.Name] = of
			}
			return true
		}
		osutil.HideFile(of.temp)
	}

	if of.err != nil {
		// We have already failed this file.
		if debug {
			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
		}
		if b.last {
			delete(p.openFiles, f.Name)
		}

		return true
	}

	p.openFiles[f.Name] = of

	switch {
	case len(b.copy) > 0:
		p.handleCopyBlock(b)
		return true

	case b.block.Size > 0:
		return p.handleRequestBlock(b)

	default:
		p.handleEmptyBlock(b)
		return true
	}
}
Example #16
0
func (p *puller) fixupDirectories() {
	var deleteDirs []string
	var changed = 0

	var walkFn = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() {
			return nil
		}

		rn, err := filepath.Rel(p.repoCfg.Directory, path)
		if err != nil {
			return nil
		}

		if rn == "." {
			return nil
		}

		if filepath.Base(rn) == ".stversions" {
			return filepath.SkipDir
		}

		cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
		if cur.Name != rn {
			// No matching dir in current list; weird
			if debug {
				l.Debugf("missing dir: %s; %v", rn, cur)
			}
			return nil
		}

		if protocol.IsDeleted(cur.Flags) {
			if debug {
				l.Debugf("queue delete dir: %v", cur)
			}

			// We queue the directories to delete since we walk the
			// tree in depth first order and need to remove the
			// directories in the opposite order.

			deleteDirs = append(deleteDirs, path)
			return nil
		}

		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
			err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
			if err != nil {
				l.Warnf("Restoring folder flags: %q: %v", path, err)
			} else {
				changed++
				if debug {
					l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
				}
			}
		}

		if cur.Modified != info.ModTime().Unix() {
			t := time.Unix(cur.Modified, 0)
			err := os.Chtimes(path, t, t)
			if err != nil {
				if runtime.GOOS != "windows" {
					// https://code.google.com/p/go/issues/detail?id=8090
					l.Warnf("Restoring folder modtime: %q: %v", path, err)
				}
			} else {
				changed++
				if debug {
					l.Debugf("restored dir modtime: %d -> %v", info.ModTime().Unix(), cur)
				}
			}
		}

		return nil
	}

	for {
		deleteDirs = nil
		changed = 0
		filepath.Walk(p.repoCfg.Directory, walkFn)

		var deleted = 0
		// Delete any queued directories
		for i := len(deleteDirs) - 1; i >= 0; i-- {
			dir := deleteDirs[i]
			if debug {
				l.Debugln("delete dir:", dir)
			}
			err := os.Remove(dir)
			if err == nil {
				deleted++
			} else if p.versioner == nil { // Failures are expected in the presence of versioning
				l.Warnln(err)
			}
		}

		if debug {
			l.Debugf("changed %d, deleted %d dirs", changed, deleted)
		}

		if changed+deleted == 0 {
			return
		}
	}
}
Example #17
0
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					var flags uint32 = protocol.FlagDirectory
					if w.IgnorePerms {
						flags |= protocol.FlagNoPermBits | 0777
					} else {
						flags |= uint32(info.Mode() & os.ModePerm)
					}
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    flags,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}
Example #18
0
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
	f := b.file

	// For directories, making sure they exist is enough.
	// Deleted directories we mark as handled and delete later.
	if protocol.IsDirectory(f.Flags) {
		if !protocol.IsDeleted(f.Flags) {
			path := filepath.Join(p.repoCfg.Directory, f.Name)
			_, err := os.Stat(path)
			if err != nil && os.IsNotExist(err) {
				if debug {
					l.Debugf("create dir: %v", f)
				}
				err = os.MkdirAll(path, 0777)
				if err != nil {
					l.Warnf("Create folder: %q: %v", path, err)
				}
			}
		} else if debug {
			l.Debugf("ignore delete dir: %v", f)
		}
		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	of, ok := p.openFiles[f.Name]
	of.done = b.last

	if !ok {
		if debug {
			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
		}

		of.availability = uint64(p.model.repoFiles[p.repoCfg.ID].Availability(f.Name))
		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))

		dirName := filepath.Dir(of.filepath)
		_, err := os.Stat(dirName)
		if err != nil {
			err = os.MkdirAll(dirName, 0777)
		}
		if err != nil {
			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}

		of.file, of.err = os.Create(of.temp)
		if of.err != nil {
			if debug {
				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
			}
			if !b.last {
				p.openFiles[f.Name] = of
			}
			return true
		}
		osutil.HideFile(of.temp)
	}

	if of.err != nil {
		// We have already failed this file.
		if debug {
			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
		}
		if b.last {
			delete(p.openFiles, f.Name)
		}

		return true
	}

	p.openFiles[f.Name] = of

	switch {
	case len(b.copy) > 0:
		p.handleCopyBlock(b)
		return true

	case b.block.Size > 0:
		return p.handleRequestBlock(b)

	default:
		p.handleEmptyBlock(b)
		return true
	}
}
Example #19
0
// ConnectionStats returns a map with connection statistics for each connected node.
func (m *Model) ConnectionStats() map[string]ConnectionInfo {
	type remoteAddrer interface {
		RemoteAddr() net.Addr
	}

	m.pmut.RLock()
	m.rmut.RLock()

	var res = make(map[string]ConnectionInfo)
	for node, conn := range m.protoConn {
		ci := ConnectionInfo{
			Statistics:    conn.Statistics(),
			ClientVersion: m.nodeVer[node],
		}
		if nc, ok := m.rawConn[node].(remoteAddrer); ok {
			ci.Address = nc.RemoteAddr().String()
		}

		var tot int64
		var have int64

		for _, repo := range m.nodeRepos[node] {
			m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
				if !protocol.IsDeleted(f.Flags) {
					var size int64
					if protocol.IsDirectory(f.Flags) {
						size = zeroEntrySize
					} else {
						size = f.Size()
					}
					tot += size
					have += size
				}
				return true
			})

			m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
				if !protocol.IsDeleted(f.Flags) {
					var size int64
					if protocol.IsDirectory(f.Flags) {
						size = zeroEntrySize
					} else {
						size = f.Size()
					}
					have -= size
				}
				return true
			})
		}

		ci.Completion = 100
		if tot != 0 {
			ci.Completion = int(100 * have / tot)
		}

		res[node.String()] = ci
	}

	m.rmut.RUnlock()
	m.pmut.RUnlock()

	in, out := protocol.TotalInOut()
	res["total"] = ConnectionInfo{
		Statistics: protocol.Statistics{
			At:            time.Now(),
			InBytesTotal:  in,
			OutBytesTotal: out,
		},
	}

	return res
}