func recurseEnumerateTree(rootDir string, c chan<- TreeItem) bool {
f, err := os.Open(rootDir)
if err != nil {
c <- TreeItem{Error: err}
return false
}
defer func() {
_ = f.Close()
}()
for {
if interrupt.IsSet() {
break
}
dirs, err := f.Readdir(128)
if err != nil && err != io.EOF {
c <- TreeItem{Error: err}
return false
}
if len(dirs) == 0 {
break
}
for _, d := range dirs {
if interrupt.IsSet() {
break
}
name := d.Name()
fullPath := filepath.Join(rootDir, name)
if d.IsDir() {
if !recurseEnumerateTree(fullPath, c) {
return false
}
} else {
c <- TreeItem{FullPath: fullPath, FileInfo: d}
}
}
}
return true
}
// Enumerates all the entries in the table. If a file or directory is found in
// the directory tree that doesn't match the expected format, it will be moved
// into the trash.
func (c *casTable) Enumerate() <-chan EnumerationEntry {
rePrefix := regexp.MustCompile(fmt.Sprintf("^[a-f0-9]{%d}$", c.prefixLength))
reRest := regexp.MustCompile(fmt.Sprintf("^[a-f0-9]{%d}$", c.hashLength-c.prefixLength))
items := make(chan EnumerationEntry)
// TODO(maruel): No need to read all at once.
go func() {
prefixes, err := readDirNames(c.casDir)
if err != nil {
items <- EnumerationEntry{Error: fmt.Errorf("Failed reading ss", c.casDir)}
} else {
for _, prefix := range prefixes {
if interrupt.IsSet() {
break
}
if prefix == trashName {
continue
}
if !rePrefix.MatchString(prefix) {
_ = c.trash.move(prefix)
c.SetFsckBit()
continue
}
// TODO(maruel): No need to read all at once.
prefixPath := filepath.Join(c.casDir, prefix)
subitems, err := readDirNames(prefixPath)
if err != nil {
items <- EnumerationEntry{Error: fmt.Errorf("Failed reading %s", prefixPath)}
c.SetFsckBit()
continue
}
for _, item := range subitems {
if !reRest.MatchString(item) {
_ = c.trash.move(filepath.Join(prefix, item))
c.SetFsckBit()
continue
}
items <- EnumerationEntry{Item: prefix + item}
}
}
}
close(items)
}()
return items
}
// Loads the list of inputs and starts the concurrent processes:
// - Enumerating the trees.
// - Updating the hash for each items in the cache.
// - Archiving items.
func (c *archiveRun) main(a DumbcasApplication, toArchiveArg string) error {
if err := c.Parse(a, true); err != nil {
return err
}
toArchive, err := filepath.Abs(toArchiveArg)
if err != nil {
return fmt.Errorf("Failed to process %s", toArchiveArg)
}
inputs, err := readFileAsStrings(toArchive)
if err != nil {
return err
}
// Make sure the file itself is archived too.
inputs = append(inputs, toArchive)
a.GetLog().Printf("Found %d entries to backup in %s", len(inputs), toArchive)
cleanupList(filepath.Dir(toArchive), inputs)
// Start the processes.
output := make(chan string)
done := make(chan bool, 3)
s := stats{out: output, done: done}
entry := s.archiveInputs(a, c.cas, s.hashInputs(a, s.enumerateInputs(inputs)))
headerWasPrinted := false
columns := []string{
"Found",
"Hashed",
"In cache",
"Archived",
"Skipped",
"Done",
}
for i := range columns {
columns[i] = fmt.Sprintf("%-19s", columns[i])
}
column := strings.TrimSpace(strings.Join(columns, ""))
errDone := errors.New("Dummy")
prevStats := s.Copy()
for err == nil {
select {
case line := <-output:
a.GetLog().Print(line)
case <-interrupt.Channel:
// Early exit. Note this as an error.
err = fmt.Errorf("Was interrupted.")
case item, ok := <-entry:
if !ok {
e := s.errors.Get()
if e != 0 {
err = fmt.Errorf("Got %d errors!", e)
} else if s.interrupted.Get() != 0 {
err = fmt.Errorf("Was interrupted.")
} else {
err = fmt.Errorf("Unexpected error.")
}
continue
}
if item != "" {
node := &dumbcaslib.Node{Entry: item, Comment: c.comment}
_, err = c.nodes.AddEntry(node, filepath.Base(toArchive))
err = errDone
} else {
e := s.errors.Get()
if e != 0 {
err = fmt.Errorf("Got %d errors!", e)
} else if s.interrupted.Get() != 0 {
err = fmt.Errorf("Was interrupted.")
} else {
err = fmt.Errorf("Unexpected error.")
}
}
case <-time.After(5 * time.Second):
nextStats := s.Copy()
if !prevStats.equals(nextStats) {
if !headerWasPrinted {
a.GetLog().Printf(column)
headerWasPrinted = true
}
prevStats = nextStats
fractionDone := float64(prevStats.bytesArchived.Get()+prevStats.bytesNotArchived.Get()) / float64(prevStats.totalSize.Get())
a.GetLog().Printf(
"%6d(%8.1fmb) %6d(%8.1fmb) %6d(%8.1fmb) %6d(%8.1fmb) %6d(%8.1fmb) %3.1f%% %d errors",
prevStats.found.Get(),
toMb(prevStats.totalSize.Get()),
prevStats.nbHashed.Get(),
toMb(prevStats.bytesHashed.Get()),
prevStats.nbNotHashed.Get(),
toMb(prevStats.bytesNotHashed.Get()),
prevStats.nbArchived.Get(),
toMb(prevStats.bytesArchived.Get()),
prevStats.nbNotArchived.Get(),
toMb(prevStats.bytesNotArchived.Get()),
100.*fractionDone,
prevStats.errors.Get())
}
}
}
if err == errDone {
err = nil
}
if interrupt.IsSet() {
fmt.Fprintf(a.GetOut(), "Was interrupted, waiting for processes to terminate.\n")
}
// Make sure all the worker threads are done. They may still be processing in
// case of interruption.
for i := 0; i < 3; i++ {
<-done
}
fmt.Fprintf(a.GetOut(), column+"\n")
fractionDone := float64(s.bytesArchived.Get()+s.bytesNotArchived.Get()) / float64(s.totalSize.Get())
fmt.Fprintf(
a.GetOut(),
"%7d(%7.1fmb) %7d(%7.1fmb) %7d(%7.1fmb) %7d(%7.1fmb) %7d(%7.1fmb) %3.1f%% %d errors\n",
s.found.Get(),
toMb(s.totalSize.Get()),
s.nbHashed.Get(),
toMb(s.bytesHashed.Get()),
s.nbNotHashed.Get(),
toMb(s.bytesNotHashed.Get()),
s.nbArchived.Get(),
toMb(s.bytesArchived.Get()),
s.nbNotArchived.Get(),
toMb(s.bytesNotArchived.Get()),
100.*fractionDone,
s.errors.Get())
return nil
}