Пример #1
0
// Returns the completion status, in percent, for the given node and repo.
func (m *Model) Completion(node protocol.NodeID, repo string) float64 {
	var tot int64
	m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
		if !protocol.IsDeleted(f.Flags) {
			var size int64
			if protocol.IsDirectory(f.Flags) {
				size = zeroEntrySize
			} else {
				size = f.Size()
			}
			tot += size
		}
		return true
	})

	var need int64
	m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
		if !protocol.IsDeleted(f.Flags) {
			var size int64
			if protocol.IsDirectory(f.Flags) {
				size = zeroEntrySize
			} else {
				size = f.Size()
			}
			need += size
		}
		return true
	})

	return 100 * (1 - float64(need)/float64(tot))
}
Пример #2
0
func hashFile(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
	for f := range inbox {
		if protocol.IsDirectory(f.Flags) || protocol.IsDeleted(f.Flags) {
			outbox <- f
			continue
		}

		fd, err := os.Open(filepath.Join(dir, f.Name))
		if err != nil {
			if debug {
				l.Debugln("open:", err)
			}
			continue
		}

		blocks, err := Blocks(fd, blockSize)
		fd.Close()

		if err != nil {
			if debug {
				l.Debugln("hash error:", f.Name, err)
			}
			continue
		}

		f.Blocks = blocks
		outbox <- f
	}
}
Пример #3
0
// Returns the completion status, in percent, for the given node and repo.
func (m *Model) Completion(node protocol.NodeID, repo string) float64 {
	var tot int64

	m.rmut.RLock()
	rf, ok := m.repoFiles[repo]
	m.rmut.RUnlock()
	if !ok {
		return 0 // Repo doesn't exist, so we hardly have any of it
	}

	rf.WithGlobal(func(f protocol.FileInfo) bool {
		if !protocol.IsDeleted(f.Flags) {
			var size int64
			if protocol.IsDirectory(f.Flags) {
				size = zeroEntrySize
			} else {
				size = f.Size()
			}
			tot += size
		}
		return true
	})

	if tot == 0 {
		return 100 // Repo is empty, so we have all of it
	}

	var need int64
	rf.WithNeed(node, func(f protocol.FileInfo) bool {
		if !protocol.IsDeleted(f.Flags) {
			var size int64
			if protocol.IsDirectory(f.Flags) {
				size = zeroEntrySize
			} else {
				size = f.Size()
			}
			need += size
		}
		return true
	})

	return 100 * (1 - float64(need)/float64(tot))
}
Пример #4
0
func (p *puller) handleEmptyBlock(b bqBlock) {
	f := b.file
	of := p.openFiles[f.Name]

	if b.last {
		if of.err == nil {
			of.file.Close()
		}
	}

	if protocol.IsDeleted(f.Flags) {
		if debug {
			l.Debugf("pull: delete %q", f.Name)
		}
		os.Remove(of.temp)

		// Ensure the file and the directory it is in is writeable so we can remove the file
		dirName := filepath.Dir(of.filepath)
		os.Chmod(of.filepath, 0666)
		if dirName != p.repoCfg.Directory {
			os.Chmod(dirName, 0777)
		}
		if p.versioner != nil {
			if debug {
				l.Debugln("pull: deleting with versioner")
			}
			if err := p.versioner.Archive(p.repoCfg.Directory, of.filepath); err == nil {
				p.model.updateLocal(p.repoCfg.ID, f)
			} else if debug {
				l.Debugln("pull: error:", err)
			}
		} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
			p.model.updateLocal(p.repoCfg.ID, f)
		}
	} else {
		if debug {
			l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name)
		}
		t := time.Unix(f.Modified, 0)
		if os.Chtimes(of.temp, t, t) != nil {
			delete(p.openFiles, f.Name)
			return
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil {
			delete(p.openFiles, f.Name)
			return
		}
		osutil.ShowFile(of.temp)
		if osutil.Rename(of.temp, of.filepath) == nil {
			p.model.updateLocal(p.repoCfg.ID, f)
		}
	}
	delete(p.openFiles, f.Name)
}
Пример #5
0
func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
	if !protocol.IsDeleted(f.Flags) {
		files++
		if !protocol.IsDirectory(f.Flags) {
			bytes += f.Size()
		} else {
			bytes += zeroEntrySize
		}
	} else {
		deleted++
		bytes += zeroEntrySize
	}
	return
}
Пример #6
0
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64, size int) ([]byte, error) {
	// Verify that the requested file exists in the local model.
	m.rmut.RLock()
	r, ok := m.repoFiles[repo]
	m.rmut.RUnlock()

	if !ok {
		l.Warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
		return nil, ErrNoSuchFile
	}

	lf := r.Get(protocol.LocalNodeID, name)
	if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
		if debug {
			l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf)
		}
		return nil, ErrInvalid
	}

	if offset > lf.Size() {
		if debug {
			l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
		}
		return nil, ErrNoSuchFile
	}

	if debug && nodeID != protocol.LocalNodeID {
		l.Debugf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
	}
	m.rmut.RLock()
	fn := filepath.Join(m.repoCfgs[repo].Directory, name)
	m.rmut.RUnlock()
	fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	buf := make([]byte, size)
	_, err = fd.ReadAt(buf, offset)
	if err != nil {
		return nil, err
	}

	return buf, nil
}
Пример #7
0
func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64 {
		var f protocol.FileInfo
		err := f.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if !protocol.IsDeleted(f.Flags) {
			if debug {
				l.Debugf("mark deleted; repo=%q node=%v name=%q", repo, protocol.NodeIDFromBytes(node), name)
			}
			ts := clock(f.LocalVersion)
			f.Blocks = nil
			f.Version = lamport.Default.Tick(f.Version)
			f.Flags |= protocol.FlagDeleted
			f.LocalVersion = ts
			batch.Put(dbi.Key(), f.MarshalXDR())
			ldbUpdateGlobal(db, batch, repo, node, nodeKeyName(dbi.Key()), f.Version)
			return ts
		}
		return 0
	})
}
Пример #8
0
func (m *Model) ScanRepoSub(repo, sub string) error {
	if p := filepath.Clean(filepath.Join(repo, sub)); !strings.HasPrefix(p, repo) {
		return errors.New("invalid subpath")
	}

	m.rmut.RLock()
	fs, ok := m.repoFiles[repo]
	dir := m.repoCfgs[repo].Directory

	ignores, _ := ignore.Load(filepath.Join(dir, ".stignore"))
	m.repoIgnores[repo] = ignores

	w := &scanner.Walker{
		Dir:          dir,
		Sub:          sub,
		Ignores:      ignores,
		BlockSize:    scanner.StandardBlockSize,
		TempNamer:    defTempNamer,
		CurrentFiler: cFiler{m, repo},
		IgnorePerms:  m.repoCfgs[repo].IgnorePerms,
	}
	m.rmut.RUnlock()
	if !ok {
		return errors.New("no such repo")
	}

	m.setState(repo, RepoScanning)
	fchan, err := w.Walk()

	if err != nil {
		return err
	}
	batchSize := 100
	batch := make([]protocol.FileInfo, 0, 00)
	for f := range fchan {
		events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
			"repo":     repo,
			"name":     f.Name,
			"modified": time.Unix(f.Modified, 0),
			"flags":    fmt.Sprintf("0%o", f.Flags),
			"size":     f.Size(),
		})
		if len(batch) == batchSize {
			fs.Update(protocol.LocalNodeID, batch)
			batch = batch[:0]
		}
		batch = append(batch, f)
	}
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	batch = batch[:0]
	// TODO: We should limit the Have scanning to start at sub
	seenPrefix := false
	fs.WithHaveTruncated(protocol.LocalNodeID, func(fi protocol.FileIntf) bool {
		f := fi.(protocol.FileInfoTruncated)
		if !strings.HasPrefix(f.Name, sub) {
			// Return true so that we keep iterating, until we get to the part
			// of the tree we are interested in. Then return false so we stop
			// iterating when we've passed the end of the subtree.
			return !seenPrefix
		}

		seenPrefix = true
		if !protocol.IsDeleted(f.Flags) {
			if f.IsInvalid() {
				return true
			}

			if len(batch) == batchSize {
				fs.Update(protocol.LocalNodeID, batch)
				batch = batch[:0]
			}

			if ignores.Match(f.Name) {
				// File has been ignored. Set invalid bit.
				nf := protocol.FileInfo{
					Name:     f.Name,
					Flags:    f.Flags | protocol.FlagInvalid,
					Modified: f.Modified,
					Version:  f.Version, // The file is still the same, so don't bump version
				}
				events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
					"repo":     repo,
					"name":     f.Name,
					"modified": time.Unix(f.Modified, 0),
					"flags":    fmt.Sprintf("0%o", f.Flags),
					"size":     f.Size(),
				})
				batch = append(batch, nf)
			} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
				// File has been deleted
				nf := protocol.FileInfo{
					Name:     f.Name,
					Flags:    f.Flags | protocol.FlagDeleted,
					Modified: f.Modified,
					Version:  lamport.Default.Tick(f.Version),
				}
				events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
					"repo":     repo,
					"name":     f.Name,
					"modified": time.Unix(f.Modified, 0),
					"flags":    fmt.Sprintf("0%o", f.Flags),
					"size":     f.Size(),
				})
				batch = append(batch, nf)
			}
		}
		return true
	})
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	m.setState(repo, RepoIdle)
	return nil
}
Пример #9
0
// handleBlock fulfills the block request by copying, ignoring or fetching
// from the network. Returns true if the block was fully handled
// synchronously, i.e. if the slot can be reused.
func (p *puller) handleBlock(b bqBlock) bool {
	f := b.file

	// For directories, making sure they exist is enough.
	// Deleted directories we mark as handled and delete later.
	if protocol.IsDirectory(f.Flags) {
		if !protocol.IsDeleted(f.Flags) {
			path := filepath.Join(p.repoCfg.Directory, f.Name)
			_, err := os.Stat(path)
			if err != nil && os.IsNotExist(err) {
				if debug {
					l.Debugf("create dir: %v", f)
				}
				err = os.MkdirAll(path, os.FileMode(f.Flags&0777))
				if err != nil {
					p.errors++
					l.Infof("mkdir: error: %q: %v", path, err)
				}
			}
		} else if debug {
			l.Debugf("ignore delete dir: %v", f)
		}
		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
		// We are supposed to copy the entire file, and then fetch nothing.
		// We don't actually need to make the copy.
		if debug {
			l.Debugln("taking shortcut:", f)
		}
		fp := filepath.Join(p.repoCfg.Directory, f.Name)
		t := time.Unix(f.Modified, 0)
		err := os.Chtimes(fp, t, t)
		if err != nil {
			l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
		}
		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
			err = os.Chmod(fp, os.FileMode(f.Flags&0777))
			if err != nil {
				l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
		}

		events.Default.Log(events.ItemStarted, map[string]string{
			"repo": p.repoCfg.ID,
			"item": f.Name,
		})

		p.model.updateLocal(p.repoCfg.ID, f)
		return true
	}

	of, ok := p.openFiles[f.Name]
	of.done = b.last

	if !ok {
		if debug {
			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
		}

		events.Default.Log(events.ItemStarted, map[string]string{
			"repo": p.repoCfg.ID,
			"item": f.Name,
		})

		of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))

		dirName := filepath.Dir(of.filepath)
		info, err := os.Stat(dirName)
		if err != nil {
			err = os.MkdirAll(dirName, 0777)
			if debug && err != nil {
				l.Debugf("mkdir: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
			}
		} else {
			// We need to make sure the directory is writeable so we can create files in it
			if dirName != p.repoCfg.Directory {
				err = os.Chmod(dirName, 0777)
				if debug && err != nil {
					l.Debugf("make writeable: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
				}
			}
			// Change it back after creating the file, to minimize the time window with incorrect permissions
			defer os.Chmod(dirName, info.Mode())
		}

		of.file, of.err = os.Create(of.temp)
		if of.err != nil {
			p.errors++
			l.Infof("create: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
			if !b.last {
				p.openFiles[f.Name] = of
			}
			return true
		}
		osutil.HideFile(of.temp)
	}

	if of.err != nil {
		// We have already failed this file.
		if debug {
			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
		}
		if b.last {
			delete(p.openFiles, f.Name)
		}

		return true
	}

	p.openFiles[f.Name] = of

	switch {
	case len(b.copy) > 0:
		p.handleCopyBlock(b)
		return true

	case b.block.Size > 0:
		return p.handleRequestBlock(b)

	default:
		p.handleEmptyBlock(b)
		return true
	}
}
Пример #10
0
// clean deletes orphaned temporary files and directories that should no
// longer exist.
func (p *puller) clean() {
	var deleteDirs []string
	var changed = 0

	var walkFn = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) {
			os.Remove(path)
		}

		if !info.IsDir() {
			return nil
		}

		rn, err := filepath.Rel(p.repoCfg.Directory, path)
		if err != nil {
			return nil
		}

		if rn == "." {
			return nil
		}

		if filepath.Base(rn) == ".stversions" {
			return filepath.SkipDir
		}

		cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
		if cur.Name != rn {
			// No matching dir in current list; weird
			if debug {
				l.Debugf("missing dir: %s; %v", rn, cur)
			}
			return nil
		}

		if protocol.IsDeleted(cur.Flags) {
			if debug {
				l.Debugf("queue delete dir: %v", cur)
			}

			// We queue the directories to delete since we walk the
			// tree in depth first order and need to remove the
			// directories in the opposite order.

			deleteDirs = append(deleteDirs, path)
			return nil
		}

		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
			err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
			if err != nil {
				l.Warnf("Restoring folder flags: %q: %v", path, err)
			} else {
				changed++
				if debug {
					l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
				}
			}
		}

		return nil
	}

	for {
		deleteDirs = nil
		changed = 0
		filepath.Walk(p.repoCfg.Directory, walkFn)

		var deleted = 0
		// Delete any queued directories
		for i := len(deleteDirs) - 1; i >= 0; i-- {
			dir := deleteDirs[i]
			if debug {
				l.Debugln("delete dir:", dir)
			}
			err := os.Remove(dir)
			if err == nil {
				deleted++
			} else {
				l.Warnln("Delete dir:", err)
			}
		}

		if debug {
			l.Debugf("changed %d, deleted %d dirs", changed, deleted)
		}

		if changed+deleted == 0 {
			return
		}
	}
}
Пример #11
0
func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) {
			l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn)
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged {
					return nil
				}
			}

			var flags uint32 = protocol.FlagDirectory
			if w.IgnorePerms {
				flags |= protocol.FlagNoPermBits | 0777
			} else {
				flags |= uint32(info.Mode() & os.ModePerm)
			}
			f := protocol.FileInfo{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
			}
			if debug {
				l.Debugln("dir:", f)
			}
			fchan <- f
			return nil
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					return nil
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}

			fchan <- protocol.FileInfo{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
			}
		}

		return nil
	}
}
Пример #12
0
func (m *Model) ScanRepoSub(repo, sub string) error {
	if p := filepath.Clean(filepath.Join(repo, sub)); !strings.HasPrefix(p, repo) {
		return errors.New("invalid subpath")
	}

	m.rmut.RLock()
	fs, ok := m.repoFiles[repo]
	dir := m.repoCfgs[repo].Directory

	w := &scanner.Walker{
		Dir:          dir,
		Sub:          sub,
		IgnoreFile:   ".stignore",
		BlockSize:    scanner.StandardBlockSize,
		TempNamer:    defTempNamer,
		CurrentFiler: cFiler{m, repo},
		IgnorePerms:  m.repoCfgs[repo].IgnorePerms,
	}
	m.rmut.RUnlock()
	if !ok {
		return errors.New("no such repo")
	}

	m.setState(repo, RepoScanning)
	fchan, err := w.Walk()

	if err != nil {
		return err
	}
	batchSize := 100
	batch := make([]protocol.FileInfo, 0, 00)
	for f := range fchan {
		events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
			"repo":     repo,
			"name":     f.Name,
			"modified": time.Unix(f.Modified, 0),
			"flags":    fmt.Sprintf("0%o", f.Flags),
			"size":     f.Size(),
		})
		if len(batch) == batchSize {
			fs.Update(protocol.LocalNodeID, batch)
			batch = batch[:0]
		}
		batch = append(batch, f)
	}
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	batch = batch[:0]
	// TODO: We should limit the Have scanning to start at sub
	seenPrefix := false
	fs.WithHaveTruncated(protocol.LocalNodeID, func(fi protocol.FileIntf) bool {
		f := fi.(protocol.FileInfoTruncated)
		if !strings.HasPrefix(f.Name, sub) {
			return !seenPrefix
		}
		seenPrefix = true
		if !protocol.IsDeleted(f.Flags) {
			if len(batch) == batchSize {
				fs.Update(protocol.LocalNodeID, batch)
				batch = batch[:0]
			}
			if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
				// File has been deleted
				nf := protocol.FileInfo{
					Name:     f.Name,
					Flags:    f.Flags | protocol.FlagDeleted,
					Modified: f.Modified,
					Version:  lamport.Default.Tick(f.Version),
				}
				events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
					"repo":     repo,
					"name":     f.Name,
					"modified": time.Unix(f.Modified, 0),
					"flags":    fmt.Sprintf("0%o", f.Flags),
					"size":     f.Size(),
				})
				batch = append(batch, nf)
			}
		}
		return true
	})
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	m.setState(repo, RepoIdle)
	return nil
}
Пример #13
0
func (m *Model) ScanRepo(repo string) error {
	m.rmut.RLock()
	fs := m.repoFiles[repo]
	dir := m.repoCfgs[repo].Directory

	w := &scanner.Walker{
		Dir:          dir,
		IgnoreFile:   ".stignore",
		BlockSize:    scanner.StandardBlockSize,
		TempNamer:    defTempNamer,
		Suppressor:   m.suppressor[repo],
		CurrentFiler: cFiler{m, repo},
		IgnorePerms:  m.repoCfgs[repo].IgnorePerms,
	}
	m.rmut.RUnlock()

	m.setState(repo, RepoScanning)
	fchan, _, err := w.Walk()

	if err != nil {
		return err
	}
	batchSize := 100
	batch := make([]protocol.FileInfo, 0, 00)
	for f := range fchan {
		events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
			"repo":     repo,
			"name":     f.Name,
			"modified": time.Unix(f.Modified, 0),
			"flags":    fmt.Sprintf("0%o", f.Flags),
			"size":     f.Size(),
		})
		if len(batch) == batchSize {
			fs.Update(protocol.LocalNodeID, batch)
			batch = batch[:0]
		}
		batch = append(batch, f)
	}
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	batch = batch[:0]
	fs.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
		if !protocol.IsDeleted(f.Flags) {
			if len(batch) == batchSize {
				fs.Update(protocol.LocalNodeID, batch)
				batch = batch[:0]
			}
			if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
				// File has been deleted
				f.Blocks = nil
				f.Flags |= protocol.FlagDeleted
				f.Version = lamport.Default.Tick(f.Version)
				f.LocalVersion = 0
				events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
					"repo":     repo,
					"name":     f.Name,
					"modified": time.Unix(f.Modified, 0),
					"flags":    fmt.Sprintf("0%o", f.Flags),
					"size":     f.Size(),
				})
				batch = append(batch, f)
			}
		}
		return true
	})
	if len(batch) > 0 {
		fs.Update(protocol.LocalNodeID, batch)
	}

	m.setState(repo, RepoIdle)
	return nil
}
Пример #14
0
func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
	start := globalKey(repo, nil)
	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
	snap, err := db.GetSnapshot()
	if err != nil {
		panic(err)
	}
	defer snap.Release()
	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
	defer dbi.Release()

	for dbi.Next() {
		var vl versionList
		err := vl.UnmarshalXDR(dbi.Value())
		if err != nil {
			panic(err)
		}
		if len(vl.versions) == 0 {
			l.Debugln(dbi.Key())
			panic("no versions?")
		}

		have := false // If we have the file, any version
		need := false // If we have a lower version of the file
		var haveVersion uint64
		for _, v := range vl.versions {
			if bytes.Compare(v.node, node) == 0 {
				have = true
				haveVersion = v.version
				need = v.version < vl.versions[0].version
				break
			}
		}

		if need || !have {
			name := globalKeyName(dbi.Key())
			fk := nodeKey(repo, vl.versions[0].node, name)
			bs, err := snap.Get(fk, nil)
			if err != nil {
				panic(err)
			}

			var gf protocol.FileInfo
			err = gf.UnmarshalXDR(bs)
			if err != nil {
				panic(err)
			}

			if protocol.IsDeleted(gf.Flags) && !have {
				// We don't need deleted files that we don't have
				continue
			}

			if debug {
				l.Debugf("need repo=%q node=%v name=%q need=%v have=%v haveV=%d globalV=%d", repo, protocol.NodeIDFromBytes(node), name, need, have, haveVersion, vl.versions[0].version)
			}

			if cont := fn(gf); !cont {
				return
			}
		}
	}
}