func findUntagged(store *storage.Storage, tx *storage.Tx, paths []string, recursive bool) error {
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not get absolute path: %v", path, err)
}
//TODO PERF no need to retrieve file: we merely need to know it exists
file, err := store.FileByPath(tx, absPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", path, err)
}
if file == nil {
relPath := _path.Rel(absPath)
fmt.Println(relPath)
}
if recursive {
entries, err := directoryEntries(path)
if err != nil {
return err
}
findUntagged(store, tx, entries, true)
}
}
return nil
}
func repairMoved(store *storage.Storage, tx *storage.Tx, missing entities.Files, searchPaths []string, pretend bool, settings entities.Settings) error {
log.Infof(2, "repairing moved files")
if len(missing) == 0 || len(searchPaths) == 0 {
// don't bother enumerating filesystem if nothing to do
return nil
}
pathsBySize, err := buildPathBySizeMap(searchPaths)
if err != nil {
return err
}
for index, dbFile := range missing {
log.Infof(2, "%v: searching for new location", dbFile.Path())
pathsOfSize := pathsBySize[dbFile.Size]
log.Infof(2, "%v: file is of size %v, identified %v files of this size", dbFile.Path(), dbFile.Size, len(pathsOfSize))
for _, candidatePath := range pathsOfSize {
candidateFile, err := store.FileByPath(tx, candidatePath)
if err != nil {
return err
}
if candidateFile != nil {
// file is already tagged
continue
}
stat, err := os.Stat(candidatePath)
if err != nil {
return fmt.Errorf("%v: could not stat file: %v", candidatePath, err)
}
fingerprint, err := fingerprint.Create(candidatePath, settings.FileFingerprintAlgorithm(), settings.DirectoryFingerprintAlgorithm(), settings.SymlinkFingerprintAlgorithm())
if err != nil {
return fmt.Errorf("%v: could not create fingerprint: %v", candidatePath, err)
}
if fingerprint == dbFile.Fingerprint {
if !pretend {
_, err := store.UpdateFile(tx, dbFile.Id, candidatePath, dbFile.Fingerprint, stat.ModTime(), dbFile.Size, dbFile.IsDir)
if err != nil {
return fmt.Errorf("%v: could not update file in database: %v", dbFile.Path(), err)
}
}
fmt.Printf("%v: updated path to %v\n", dbFile.Path(), candidatePath)
missing[index] = nil
break
}
}
}
return nil
}
func untagPathsAll(store *storage.Storage, tx *storage.Tx, paths []string, recursive, followSymlinks bool) (error, warnings) {
warnings := make(warnings, 0, 10)
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not get absolute path: %v", path, err), warnings
}
log.Infof(2, "%v: resolving path", path)
stat, err := os.Lstat(absPath)
if err != nil {
switch {
case os.IsNotExist(err), os.IsPermission(err):
// ignore
default:
return err, nil
}
} else if stat.Mode()&os.ModeSymlink != 0 && followSymlinks {
absPath, err = _path.Dereference(absPath)
if err != nil {
return err, nil
}
}
file, err := store.FileByPath(tx, absPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", path, err), warnings
}
if file == nil {
warnings = append(warnings, fmt.Sprintf("%v: file is not tagged.", path))
continue
}
log.Infof(2, "%v: removing all tags.", path)
if err := store.DeleteFileTagsByFileId(tx, file.Id); err != nil {
return fmt.Errorf("%v: could not remove file's tags: %v", path, err), warnings
}
if recursive {
childFiles, err := store.FilesByDirectory(tx, file.Path())
if err != nil {
return fmt.Errorf("%v: could not retrieve files for directory: %v", path, err), warnings
}
for _, childFile := range childFiles {
if err := store.DeleteFileTagsByFileId(tx, childFile.Id); err != nil {
return fmt.Errorf("%v: could not remove file's tags: %v", childFile.Path(), err), warnings
}
}
}
}
return nil, warnings
}
func tagFrom(store *storage.Storage, tx *storage.Tx, fromPath string, paths []string, explicit, recursive, force, followSymlinks bool) (error, warnings) {
log.Infof(2, "loading settings")
settings, err := store.Settings(tx)
if err != nil {
return fmt.Errorf("could not retrieve settings: %v", err), nil
}
stat, err := os.Lstat(fromPath)
if err != nil {
return err, nil
}
if stat.Mode()&os.ModeSymlink != 0 && followSymlinks {
fromPath, err = _path.Dereference(fromPath)
if err != nil {
return err, nil
}
}
file, err := store.FileByPath(tx, fromPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", fromPath, err), nil
}
if file == nil {
return fmt.Errorf("%v: path is not tagged", fromPath), nil
}
fileTags, err := store.FileTagsByFileId(tx, file.Id, true)
if err != nil {
return fmt.Errorf("%v: could not retrieve filetags: %v", fromPath, err), nil
}
pairs := make([]entities.TagIdValueIdPair, len(fileTags))
for index, fileTag := range fileTags {
pairs[index] = entities.TagIdValueIdPair{fileTag.TagId, fileTag.ValueId}
}
warnings := make(warnings, 0, 10)
for _, path := range paths {
if err := tagPath(store, tx, path, pairs, explicit, recursive, force, followSymlinks, settings.FileFingerprintAlgorithm(), settings.DirectoryFingerprintAlgorithm(), settings.SymlinkFingerprintAlgorithm(), settings.ReportDuplicates()); err != nil {
switch {
case os.IsPermission(err):
warnings = append(warnings, fmt.Sprintf("%v: permisison denied", path))
case os.IsNotExist(err):
warnings = append(warnings, fmt.Sprintf("%v: no such file", path))
default:
return fmt.Errorf("%v: could not stat file: %v", path, err), warnings
}
}
}
return nil, warnings
}
func manualRepair(store *storage.Storage, tx *storage.Tx, fromPath, toPath string, pretend bool) error {
absFromPath, err := filepath.Abs(fromPath)
if err != nil {
return fmt.Errorf("%v: could not determine absolute path", err)
}
absToPath, err := filepath.Abs(toPath)
if err != nil {
return fmt.Errorf("%v: could not determine absolute path", err)
}
log.Infof(2, "retrieving files under '%v' from the database", fromPath)
dbFile, err := store.FileByPath(tx, absFromPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", fromPath, err)
}
if dbFile != nil {
log.Infof(2, "%v: updating to %v", fromPath, toPath)
if !pretend {
if err := manualRepairFile(store, tx, dbFile, absToPath); err != nil {
return err
}
}
}
dbFiles, err := store.FilesByDirectory(tx, absFromPath)
if err != nil {
return fmt.Errorf("could not retrieve files from storage: %v", err)
}
for _, dbFile = range dbFiles {
relFileFromPath := _path.Rel(dbFile.Path())
absFileToPath := strings.Replace(dbFile.Path(), absFromPath, absToPath, 1)
relFileToPath := _path.Rel(absFileToPath)
log.Infof(2, "%v: updating to %v", relFileFromPath, relFileToPath)
if !pretend {
if err := manualRepairFile(store, tx, dbFile, absFileToPath); err != nil {
return err
}
}
}
return nil
}
func statusPaths(store *storage.Storage, tx *storage.Tx, paths []string, dirOnly bool) (*StatusReport, error) {
report := NewReport()
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("%v: could not get absolute path: %v", path, err)
}
file, err := store.FileByPath(tx, absPath)
if err != nil {
return nil, fmt.Errorf("%v: could not retrieve file: %v", path, err)
}
if file != nil {
err = statusCheckFile(file, report)
if err != nil {
return nil, err
}
}
if !dirOnly {
log.Infof(2, "%v: retrieving files from database.", path)
files, err := store.FilesByDirectory(tx, absPath)
if err != nil {
return nil, fmt.Errorf("%v: could not retrieve files for directory: %v", path, err)
}
err = statusCheckFiles(files, report)
if err != nil {
return nil, err
}
}
err = findNewFiles(absPath, report, dirOnly)
if err != nil {
return nil, err
}
}
return report, nil
}
func untagPathsAll(store *storage.Storage, tx *storage.Tx, paths []string, recursive bool) (error, warnings) {
warnings := make(warnings, 0, 10)
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not get absolute path: %v", path, err), warnings
}
file, err := store.FileByPath(tx, absPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", path, err), warnings
}
if file == nil {
warnings = append(warnings, fmt.Sprintf("%v: file is not tagged.", path))
continue
}
log.Infof(2, "%v: removing all tags.", file.Path())
if err := store.DeleteFileTagsByFileId(tx, file.Id); err != nil {
return fmt.Errorf("%v: could not remove file's tags: %v", file.Path(), err), warnings
}
if recursive {
childFiles, err := store.FilesByDirectory(tx, file.Path())
if err != nil {
return fmt.Errorf("%v: could not retrieve files for directory: %v", file.Path()), warnings
}
for _, childFile := range childFiles {
if err := store.DeleteFileTagsByFileId(tx, childFile.Id); err != nil {
return fmt.Errorf("%v: could not remove file's tags: %v", childFile.Path(), err), warnings
}
}
}
}
return nil, warnings
}
func findUntaggedFunc(store *storage.Storage, tx *storage.Tx, paths []string, recursive, followSymlinks bool, action func(absPath string)) error {
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not get absolute path: %v", path, err)
}
if followSymlinks {
log.Infof(2, "%v: resolving path", path)
absPath, err = _path.Dereference(absPath)
if err != nil {
return fmt.Errorf("%v: could not dereference path: %v", path, err)
}
}
//TODO PERF no need to retrieve file: we merely need to know it exists
file, err := store.FileByPath(tx, absPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", path, err)
}
if file == nil {
action(absPath)
}
if recursive {
entries, err := directoryEntries(path)
if err != nil {
return err
}
findUntaggedFunc(store, tx, entries, true, followSymlinks, action)
}
}
return nil
}
func listTagsForPaths(store *storage.Storage, tx *storage.Tx, paths []string, showCount, onePerLine, explicitOnly, printPath, colour bool) (error, warnings) {
warnings := make(warnings, 0, 10)
printPath = printPath || len(paths) > 1 || !stdoutIsCharDevice()
for index, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return err, warnings
}
log.Infof(2, "%v: retrieving tags.", path)
file, err := store.FileByPath(tx, absPath)
if err != nil {
warnings = append(warnings, err.Error())
continue
}
var tagNames []string
if file != nil {
tagNames, err = tagNamesForFile(store, tx, file.Id, explicitOnly, colour)
if err != nil {
return err, warnings
}
} else {
_, err := os.Stat(absPath)
if err != nil {
switch {
case os.IsPermission(err):
warnings = append(warnings, fmt.Sprintf("%v: permission denied", path))
continue
case os.IsNotExist(err):
warnings = append(warnings, fmt.Sprintf("%v: no such file", path))
continue
default:
return fmt.Errorf("%v: could not stat file: %v", path, err), warnings
}
}
}
switch {
case showCount:
if printPath {
fmt.Print(path + ": ")
}
fmt.Println(strconv.Itoa(len(tagNames)))
case onePerLine:
if index > 0 {
fmt.Println()
}
if printPath {
fmt.Println(path + ":")
}
for _, tagName := range tagNames {
fmt.Println(tagName)
}
default:
if printPath {
fmt.Print(path + ":")
for _, tagName := range tagNames {
fmt.Print(" " + tagName)
}
fmt.Println()
} else {
terminal.PrintColumns(tagNames)
}
}
}
return nil, warnings
}
func statusPaths(store *storage.Storage, tx *storage.Tx, paths []string, dirOnly, followSymlinks bool) (*StatusReport, error) {
report := NewReport()
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("%v: could not get absolute path: %v", path, err)
}
log.Infof(2, "%v: resolving file", path)
resolvedPath := absPath
stat, err := os.Lstat(absPath)
if err != nil {
switch {
case os.IsNotExist(err), os.IsPermission(err):
stat = emptyStat{}
default:
return nil, fmt.Errorf("%v: could not stat path: %v", path, err)
}
} else if stat.Mode()&os.ModeSymlink != 0 {
resolvedPath, err = _path.Dereference(absPath)
if err != nil {
return nil, fmt.Errorf("%v: could not dereference symbolic link: %v", path, err)
}
}
log.Infof(2, "%v: checking file in database", path)
file, err := store.FileByPath(tx, resolvedPath)
if err != nil {
return nil, fmt.Errorf("%v: could not retrieve file: %v", path, err)
}
if file != nil {
err = statusCheckFile(absPath, file, report)
if err != nil {
return nil, err
}
}
if !dirOnly && (stat.Mode()&os.ModeSymlink == 0 || followSymlinks) {
log.Infof(2, "%v: retrieving files from database.", path)
files, err := store.FilesByDirectory(tx, resolvedPath)
if err != nil {
return nil, fmt.Errorf("%v: could not retrieve files for directory: %v", path, err)
}
err = statusCheckFiles(files, report)
if err != nil {
return nil, err
}
}
err = findNewFiles(absPath, report, dirOnly, followSymlinks)
if err != nil {
return nil, err
}
}
return report, nil
}
func tagPath(store *storage.Storage, tx *storage.Tx, path string, pairs []entities.TagIdValueIdPair, explicit, recursive, force bool, fileFingerprintAlg, dirFingerprintAlg, symlinkFingerprintAlg string, reportDuplicates bool) error {
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not get absolute path: %v", path, err)
}
stat, err := os.Stat(path)
if err != nil {
switch {
case os.IsNotExist(err), os.IsPermission(err):
if !force {
return err
} else {
stat = emptyStat{}
}
default:
return err
}
}
log.Infof(2, "%v: checking if file exists in database", path)
file, err := store.FileByPath(tx, absPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", path, err)
}
if file == nil {
log.Infof(2, "%v: creating fingerprint", path)
fp, err := fingerprint.Create(path, fileFingerprintAlg, dirFingerprintAlg, symlinkFingerprintAlg)
if err != nil {
if !force || !(os.IsNotExist(err) || os.IsPermission(err)) {
return fmt.Errorf("%v: could not create fingerprint: %v", path, err)
}
}
if fp != fingerprint.Empty && reportDuplicates {
log.Infof(2, "%v: checking for duplicates", path)
count, err := store.FileCountByFingerprint(tx, fp)
if err != nil {
return fmt.Errorf("%v: could not identify duplicates: %v", path, err)
}
if count != 0 {
log.Warnf("'%v' is a duplicate", path)
}
}
log.Infof(2, "%v: adding file", path)
file, err = store.AddFile(tx, path, fp, stat.ModTime(), int64(stat.Size()), stat.IsDir())
if err != nil {
return fmt.Errorf("%v: could not add file to database: %v", path, err)
}
}
if !explicit {
pairs, err = removeAlreadyAppliedTagValuePairs(store, tx, pairs, file)
if err != nil {
return fmt.Errorf("%v: could not remove applied tags: %v", path, err)
}
}
log.Infof(2, "%v: applying tags.", path)
for _, pair := range pairs {
if _, err = store.AddFileTag(tx, file.Id, pair.TagId, pair.ValueId); err != nil {
return fmt.Errorf("%v: could not apply tags: %v", file.Path(), err)
}
}
if recursive && stat.IsDir() {
if err = tagRecursively(store, tx, path, pairs, explicit, force, fileFingerprintAlg, dirFingerprintAlg, symlinkFingerprintAlg, reportDuplicates); err != nil {
return err
}
}
return nil
}
func untagPaths(store *storage.Storage, tx *storage.Tx, paths, tagArgs []string, recursive, followSymlinks bool) (error, warnings) {
warnings := make(warnings, 0, 10)
files := make(entities.Files, 0, len(paths))
for _, path := range paths {
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not get absolute path: %v", path, err), warnings
}
log.Infof(2, "%v: resolving path", path)
stat, err := os.Lstat(absPath)
if err != nil {
switch {
case os.IsNotExist(err), os.IsPermission(err):
// ignore
default:
return err, nil
}
} else if stat.Mode()&os.ModeSymlink != 0 && followSymlinks {
absPath, err = _path.Dereference(absPath)
if err != nil {
return err, nil
}
}
file, err := store.FileByPath(tx, absPath)
if err != nil {
return fmt.Errorf("%v: could not retrieve file: %v", path, err), warnings
}
if file == nil {
warnings = append(warnings, fmt.Sprintf("%v: file is not tagged", path))
continue
}
files = append(files, file)
if recursive {
childFiles, err := store.FilesByDirectory(tx, file.Path())
if err != nil {
return fmt.Errorf("%v: could not retrieve files for directory: %v", file.Path()), warnings
}
files = append(files, childFiles...)
}
}
for _, tagArg := range tagArgs {
tagName, valueName := parseTagEqValueName(tagArg)
tag, err := store.TagByName(tx, tagName)
if err != nil {
return fmt.Errorf("could not retrieve tag '%v': %v", tagName, err), warnings
}
if tag == nil {
warnings = append(warnings, fmt.Sprintf("no such tag '%v'", tagName))
continue
}
value, err := store.ValueByName(tx, valueName)
if err != nil {
return fmt.Errorf("could not retrieve value '%v': %v", valueName, err), warnings
}
if value == nil {
warnings = append(warnings, fmt.Sprintf("no such value '%v'", valueName))
continue
}
for _, file := range files {
if err := store.DeleteFileTag(tx, file.Id, tag.Id, value.Id); err != nil {
switch err.(type) {
case storage.FileTagDoesNotExist:
exists, err := store.FileTagExists(tx, file.Id, tag.Id, value.Id, false)
if err != nil {
return fmt.Errorf("could not check if tag exists: %v", err), warnings
}
if exists {
if value.Id != 0 {
warnings = append(warnings, fmt.Sprintf("%v: cannot remove '%v=%v': delete implication to remove this tag.", file.Path(), tag.Name, value.Name))
} else {
warnings = append(warnings, fmt.Sprintf("%v: cannot remove '%v': delete implication to remove this tag.", file.Path(), tag.Name))
}
} else {
if value.Id != 0 {
warnings = append(warnings, fmt.Sprintf("%v: file is not tagged '%v=%v'.", file.Path(), tag.Name, value.Name))
} else {
warnings = append(warnings, fmt.Sprintf("%v: file is not tagged '%v'.", file.Path(), tag.Name))
}
}
default:
return fmt.Errorf("%v: could not remove tag '%v', value '%v': %v", file.Path(), tag.Name, value.Name, err), warnings
}
}
}
}
return nil, warnings
}
func fullRepair(store *storage.Storage, tx *storage.Tx, searchPaths []string, limitPath string, removeMissing, recalcUnmodified, rationalize, pretend bool) error {
absLimitPath := ""
if limitPath != "" {
var err error
absLimitPath, err = filepath.Abs(limitPath)
if err != nil {
return fmt.Errorf("%v: could not determine absolute path", err)
}
}
settings, err := store.Settings(tx)
if err != nil {
return err
}
log.Infof(2, "retrieving files under '%v' from the database", absLimitPath)
dbFiles, err := store.FilesByDirectory(tx, absLimitPath)
if err != nil {
return fmt.Errorf("could not retrieve files from storage: %v", err)
}
dbFile, err := store.FileByPath(tx, absLimitPath)
if err != nil {
return fmt.Errorf("could not retrieve file from storage: %v", err)
}
if dbFile != nil {
dbFiles = append(dbFiles, dbFile)
}
log.Infof(2, "retrieved %v files from the database for path '%v'", len(dbFiles), absLimitPath)
unmodfied, modified, missing := determineStatuses(dbFiles)
if recalcUnmodified {
if err = repairUnmodified(store, tx, unmodfied, pretend, settings); err != nil {
return err
}
}
if err = repairModified(store, tx, modified, pretend, settings); err != nil {
return err
}
if err = repairMoved(store, tx, missing, searchPaths, pretend, settings); err != nil {
return err
}
if err = repairMissing(store, tx, missing, pretend, removeMissing); err != nil {
return err
}
if err = deleteUntaggedFiles(store, tx, dbFiles); err != nil {
return err
}
if rationalize {
if err = rationalizeFileTags(store, tx, dbFiles); err != nil {
return err
}
}
return nil
}