Ejemplo n.º 1
0
func recurseEnumerateTree(rootDir string, c chan<- TreeItem) bool {
	f, err := os.Open(rootDir)
	if err != nil {
		c <- TreeItem{Error: err}
		return false
	}
	defer func() {
		_ = f.Close()
	}()
	for {
		if interrupt.IsSet() {
			break
		}
		dirs, err := f.Readdir(128)
		if err != nil && err != io.EOF {
			c <- TreeItem{Error: err}
			return false
		}
		if len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			if interrupt.IsSet() {
				break
			}
			name := d.Name()
			fullPath := filepath.Join(rootDir, name)
			if d.IsDir() {
				if !recurseEnumerateTree(fullPath, c) {
					return false
				}
			} else {
				c <- TreeItem{FullPath: fullPath, FileInfo: d}
			}
		}
	}
	return true
}
Ejemplo n.º 2
0
// Enumerates all the entries in the table. If a file or directory is found in
// the directory tree that doesn't match the expected format, it will be moved
// into the trash.
func (c *casTable) Enumerate() <-chan EnumerationEntry {
	rePrefix := regexp.MustCompile(fmt.Sprintf("^[a-f0-9]{%d}$", c.prefixLength))
	reRest := regexp.MustCompile(fmt.Sprintf("^[a-f0-9]{%d}$", c.hashLength-c.prefixLength))
	items := make(chan EnumerationEntry)

	// TODO(maruel): No need to read all at once.
	go func() {
		prefixes, err := readDirNames(c.casDir)
		if err != nil {
			items <- EnumerationEntry{Error: fmt.Errorf("Failed reading ss", c.casDir)}
		} else {
			for _, prefix := range prefixes {
				if interrupt.IsSet() {
					break
				}
				if prefix == trashName {
					continue
				}
				if !rePrefix.MatchString(prefix) {
					_ = c.trash.move(prefix)
					c.SetFsckBit()
					continue
				}
				// TODO(maruel): No need to read all at once.
				prefixPath := filepath.Join(c.casDir, prefix)
				subitems, err := readDirNames(prefixPath)
				if err != nil {
					items <- EnumerationEntry{Error: fmt.Errorf("Failed reading %s", prefixPath)}
					c.SetFsckBit()
					continue
				}
				for _, item := range subitems {
					if !reRest.MatchString(item) {
						_ = c.trash.move(filepath.Join(prefix, item))
						c.SetFsckBit()
						continue
					}
					items <- EnumerationEntry{Item: prefix + item}
				}
			}
		}
		close(items)
	}()
	return items
}
Ejemplo n.º 3
0
// Loads the list of inputs and starts the concurrent processes:
// - Enumerating the trees.
// - Updating the hash for each items in the cache.
// - Archiving items.
func (c *archiveRun) main(a DumbcasApplication, toArchiveArg string) error {
	if err := c.Parse(a, true); err != nil {
		return err
	}

	toArchive, err := filepath.Abs(toArchiveArg)
	if err != nil {
		return fmt.Errorf("Failed to process %s", toArchiveArg)
	}

	inputs, err := readFileAsStrings(toArchive)
	if err != nil {
		return err
	}
	// Make sure the file itself is archived too.
	inputs = append(inputs, toArchive)
	a.GetLog().Printf("Found %d entries to backup in %s", len(inputs), toArchive)
	cleanupList(filepath.Dir(toArchive), inputs)

	// Start the processes.
	output := make(chan string)
	done := make(chan bool, 3)
	s := stats{out: output, done: done}
	entry := s.archiveInputs(a, c.cas, s.hashInputs(a, s.enumerateInputs(inputs)))

	headerWasPrinted := false
	columns := []string{
		"Found",
		"Hashed",
		"In cache",
		"Archived",
		"Skipped",
		"Done",
	}
	for i := range columns {
		columns[i] = fmt.Sprintf("%-19s", columns[i])
	}
	column := strings.TrimSpace(strings.Join(columns, ""))

	errDone := errors.New("Dummy")
	prevStats := s.Copy()
	for err == nil {
		select {
		case line := <-output:
			a.GetLog().Print(line)
		case <-interrupt.Channel:
			// Early exit. Note this as an error.
			err = fmt.Errorf("Was interrupted.")
		case item, ok := <-entry:
			if !ok {
				e := s.errors.Get()
				if e != 0 {
					err = fmt.Errorf("Got %d errors!", e)
				} else if s.interrupted.Get() != 0 {
					err = fmt.Errorf("Was interrupted.")
				} else {
					err = fmt.Errorf("Unexpected error.")
				}
				continue
			}
			if item != "" {
				node := &dumbcaslib.Node{Entry: item, Comment: c.comment}
				_, err = c.nodes.AddEntry(node, filepath.Base(toArchive))
				err = errDone
			} else {
				e := s.errors.Get()
				if e != 0 {
					err = fmt.Errorf("Got %d errors!", e)
				} else if s.interrupted.Get() != 0 {
					err = fmt.Errorf("Was interrupted.")
				} else {
					err = fmt.Errorf("Unexpected error.")
				}
			}
		case <-time.After(5 * time.Second):
			nextStats := s.Copy()
			if !prevStats.equals(nextStats) {
				if !headerWasPrinted {
					a.GetLog().Printf(column)
					headerWasPrinted = true
				}
				prevStats = nextStats
				fractionDone := float64(prevStats.bytesArchived.Get()+prevStats.bytesNotArchived.Get()) / float64(prevStats.totalSize.Get())
				a.GetLog().Printf(
					"%6d(%8.1fmb) %6d(%8.1fmb) %6d(%8.1fmb) %6d(%8.1fmb) %6d(%8.1fmb) %3.1f%% %d errors",
					prevStats.found.Get(),
					toMb(prevStats.totalSize.Get()),
					prevStats.nbHashed.Get(),
					toMb(prevStats.bytesHashed.Get()),
					prevStats.nbNotHashed.Get(),
					toMb(prevStats.bytesNotHashed.Get()),
					prevStats.nbArchived.Get(),
					toMb(prevStats.bytesArchived.Get()),
					prevStats.nbNotArchived.Get(),
					toMb(prevStats.bytesNotArchived.Get()),
					100.*fractionDone,
					prevStats.errors.Get())
			}
		}
	}
	if err == errDone {
		err = nil
	}
	if interrupt.IsSet() {
		fmt.Fprintf(a.GetOut(), "Was interrupted, waiting for processes to terminate.\n")
	}
	// Make sure all the worker threads are done. They may still be processing in
	// case of interruption.
	for i := 0; i < 3; i++ {
		<-done
	}
	fmt.Fprintf(a.GetOut(), column+"\n")
	fractionDone := float64(s.bytesArchived.Get()+s.bytesNotArchived.Get()) / float64(s.totalSize.Get())
	fmt.Fprintf(
		a.GetOut(),
		"%7d(%7.1fmb) %7d(%7.1fmb) %7d(%7.1fmb) %7d(%7.1fmb) %7d(%7.1fmb) %3.1f%% %d errors\n",
		s.found.Get(),
		toMb(s.totalSize.Get()),
		s.nbHashed.Get(),
		toMb(s.bytesHashed.Get()),
		s.nbNotHashed.Get(),
		toMb(s.bytesNotHashed.Get()),
		s.nbArchived.Get(),
		toMb(s.bytesArchived.Get()),
		s.nbNotArchived.Get(),
		toMb(s.bytesNotArchived.Get()),
		100.*fractionDone,
		s.errors.Get())
	return nil
}