Exemple #1
0
func findUntagged(store *storage.Storage, tx *storage.Tx, paths []string, recursive bool) error {
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return fmt.Errorf("%v: could not get absolute path: %v", path, err)
		}

		//TODO PERF no need to retrieve file: we merely need to know it exists
		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			return fmt.Errorf("%v: could not retrieve file: %v", path, err)
		}
		if file == nil {
			relPath := _path.Rel(absPath)
			fmt.Println(relPath)
		}

		if recursive {
			entries, err := directoryEntries(path)
			if err != nil {
				return err
			}

			findUntagged(store, tx, entries, true)
		}
	}

	return nil
}
Exemple #2
0
func repairMoved(store *storage.Storage, tx *storage.Tx, missing entities.Files, searchPaths []string, pretend bool, settings entities.Settings) error {
	log.Infof(2, "repairing moved files")

	if len(missing) == 0 || len(searchPaths) == 0 {
		// don't bother enumerating filesystem if nothing to do
		return nil
	}

	pathsBySize, err := buildPathBySizeMap(searchPaths)
	if err != nil {
		return err
	}

	for index, dbFile := range missing {
		log.Infof(2, "%v: searching for new location", dbFile.Path())

		pathsOfSize := pathsBySize[dbFile.Size]
		log.Infof(2, "%v: file is of size %v, identified %v files of this size", dbFile.Path(), dbFile.Size, len(pathsOfSize))

		for _, candidatePath := range pathsOfSize {
			candidateFile, err := store.FileByPath(tx, candidatePath)
			if err != nil {
				return err
			}
			if candidateFile != nil {
				// file is already tagged
				continue
			}

			stat, err := os.Stat(candidatePath)
			if err != nil {
				return fmt.Errorf("%v: could not stat file: %v", candidatePath, err)
			}

			fingerprint, err := fingerprint.Create(candidatePath, settings.FileFingerprintAlgorithm(), settings.DirectoryFingerprintAlgorithm(), settings.SymlinkFingerprintAlgorithm())
			if err != nil {
				return fmt.Errorf("%v: could not create fingerprint: %v", candidatePath, err)
			}

			if fingerprint == dbFile.Fingerprint {
				if !pretend {
					_, err := store.UpdateFile(tx, dbFile.Id, candidatePath, dbFile.Fingerprint, stat.ModTime(), dbFile.Size, dbFile.IsDir)
					if err != nil {
						return fmt.Errorf("%v: could not update file in database: %v", dbFile.Path(), err)
					}
				}

				fmt.Printf("%v: updated path to %v\n", dbFile.Path(), candidatePath)

				missing[index] = nil

				break
			}
		}
	}

	return nil
}
Exemple #3
0
func untagPathsAll(store *storage.Storage, tx *storage.Tx, paths []string, recursive, followSymlinks bool) (error, warnings) {
	warnings := make(warnings, 0, 10)

	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return fmt.Errorf("%v: could not get absolute path: %v", path, err), warnings
		}

		log.Infof(2, "%v: resolving path", path)

		stat, err := os.Lstat(absPath)
		if err != nil {
			switch {
			case os.IsNotExist(err), os.IsPermission(err):
				// ignore
			default:
				return err, nil
			}
		} else if stat.Mode()&os.ModeSymlink != 0 && followSymlinks {
			absPath, err = _path.Dereference(absPath)
			if err != nil {
				return err, nil
			}
		}

		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			return fmt.Errorf("%v: could not retrieve file: %v", path, err), warnings
		}
		if file == nil {
			warnings = append(warnings, fmt.Sprintf("%v: file is not tagged.", path))
			continue
		}

		log.Infof(2, "%v: removing all tags.", path)

		if err := store.DeleteFileTagsByFileId(tx, file.Id); err != nil {
			return fmt.Errorf("%v: could not remove file's tags: %v", path, err), warnings
		}

		if recursive {
			childFiles, err := store.FilesByDirectory(tx, file.Path())
			if err != nil {
				return fmt.Errorf("%v: could not retrieve files for directory: %v", path, err), warnings
			}

			for _, childFile := range childFiles {
				if err := store.DeleteFileTagsByFileId(tx, childFile.Id); err != nil {
					return fmt.Errorf("%v: could not remove file's tags: %v", childFile.Path(), err), warnings
				}
			}
		}
	}

	return nil, warnings
}
Exemple #4
0
func tagFrom(store *storage.Storage, tx *storage.Tx, fromPath string, paths []string, explicit, recursive, force, followSymlinks bool) (error, warnings) {
	log.Infof(2, "loading settings")

	settings, err := store.Settings(tx)
	if err != nil {
		return fmt.Errorf("could not retrieve settings: %v", err), nil
	}

	stat, err := os.Lstat(fromPath)
	if err != nil {
		return err, nil
	}
	if stat.Mode()&os.ModeSymlink != 0 && followSymlinks {
		fromPath, err = _path.Dereference(fromPath)
		if err != nil {
			return err, nil
		}
	}

	file, err := store.FileByPath(tx, fromPath)
	if err != nil {
		return fmt.Errorf("%v: could not retrieve file: %v", fromPath, err), nil
	}
	if file == nil {
		return fmt.Errorf("%v: path is not tagged", fromPath), nil
	}

	fileTags, err := store.FileTagsByFileId(tx, file.Id, true)
	if err != nil {
		return fmt.Errorf("%v: could not retrieve filetags: %v", fromPath, err), nil
	}

	pairs := make([]entities.TagIdValueIdPair, len(fileTags))
	for index, fileTag := range fileTags {
		pairs[index] = entities.TagIdValueIdPair{fileTag.TagId, fileTag.ValueId}
	}

	warnings := make(warnings, 0, 10)

	for _, path := range paths {
		if err := tagPath(store, tx, path, pairs, explicit, recursive, force, followSymlinks, settings.FileFingerprintAlgorithm(), settings.DirectoryFingerprintAlgorithm(), settings.SymlinkFingerprintAlgorithm(), settings.ReportDuplicates()); err != nil {
			switch {
			case os.IsPermission(err):
				warnings = append(warnings, fmt.Sprintf("%v: permisison denied", path))
			case os.IsNotExist(err):
				warnings = append(warnings, fmt.Sprintf("%v: no such file", path))
			default:
				return fmt.Errorf("%v: could not stat file: %v", path, err), warnings
			}
		}
	}

	return nil, warnings
}
Exemple #5
0
func manualRepair(store *storage.Storage, tx *storage.Tx, fromPath, toPath string, pretend bool) error {
	absFromPath, err := filepath.Abs(fromPath)
	if err != nil {
		return fmt.Errorf("%v: could not determine absolute path", err)
	}

	absToPath, err := filepath.Abs(toPath)
	if err != nil {
		return fmt.Errorf("%v: could not determine absolute path", err)
	}

	log.Infof(2, "retrieving files under '%v' from the database", fromPath)

	dbFile, err := store.FileByPath(tx, absFromPath)
	if err != nil {
		return fmt.Errorf("%v: could not retrieve file: %v", fromPath, err)
	}

	if dbFile != nil {
		log.Infof(2, "%v: updating to %v", fromPath, toPath)

		if !pretend {
			if err := manualRepairFile(store, tx, dbFile, absToPath); err != nil {
				return err
			}
		}
	}

	dbFiles, err := store.FilesByDirectory(tx, absFromPath)
	if err != nil {
		return fmt.Errorf("could not retrieve files from storage: %v", err)
	}

	for _, dbFile = range dbFiles {
		relFileFromPath := _path.Rel(dbFile.Path())
		absFileToPath := strings.Replace(dbFile.Path(), absFromPath, absToPath, 1)
		relFileToPath := _path.Rel(absFileToPath)

		log.Infof(2, "%v: updating to %v", relFileFromPath, relFileToPath)

		if !pretend {
			if err := manualRepairFile(store, tx, dbFile, absFileToPath); err != nil {
				return err
			}
		}
	}

	return nil
}
Exemple #6
0
func statusPaths(store *storage.Storage, tx *storage.Tx, paths []string, dirOnly bool) (*StatusReport, error) {
	report := NewReport()

	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return nil, fmt.Errorf("%v: could not get absolute path: %v", path, err)
		}

		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			return nil, fmt.Errorf("%v: could not retrieve file: %v", path, err)
		}
		if file != nil {
			err = statusCheckFile(file, report)
			if err != nil {
				return nil, err
			}
		}

		if !dirOnly {
			log.Infof(2, "%v: retrieving files from database.", path)

			files, err := store.FilesByDirectory(tx, absPath)
			if err != nil {
				return nil, fmt.Errorf("%v: could not retrieve files for directory: %v", path, err)
			}

			err = statusCheckFiles(files, report)
			if err != nil {
				return nil, err
			}
		}

		err = findNewFiles(absPath, report, dirOnly)
		if err != nil {
			return nil, err
		}
	}

	return report, nil
}
Exemple #7
0
func untagPathsAll(store *storage.Storage, tx *storage.Tx, paths []string, recursive bool) (error, warnings) {
	warnings := make(warnings, 0, 10)

	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return fmt.Errorf("%v: could not get absolute path: %v", path, err), warnings
		}

		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			return fmt.Errorf("%v: could not retrieve file: %v", path, err), warnings
		}
		if file == nil {
			warnings = append(warnings, fmt.Sprintf("%v: file is not tagged.", path))
			continue
		}

		log.Infof(2, "%v: removing all tags.", file.Path())

		if err := store.DeleteFileTagsByFileId(tx, file.Id); err != nil {
			return fmt.Errorf("%v: could not remove file's tags: %v", file.Path(), err), warnings
		}

		if recursive {
			childFiles, err := store.FilesByDirectory(tx, file.Path())
			if err != nil {
				return fmt.Errorf("%v: could not retrieve files for directory: %v", file.Path()), warnings
			}

			for _, childFile := range childFiles {
				if err := store.DeleteFileTagsByFileId(tx, childFile.Id); err != nil {
					return fmt.Errorf("%v: could not remove file's tags: %v", childFile.Path(), err), warnings
				}
			}
		}
	}

	return nil, warnings
}
Exemple #8
0
func findUntaggedFunc(store *storage.Storage, tx *storage.Tx, paths []string, recursive, followSymlinks bool, action func(absPath string)) error {
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return fmt.Errorf("%v: could not get absolute path: %v", path, err)
		}

		if followSymlinks {
			log.Infof(2, "%v: resolving path", path)

			absPath, err = _path.Dereference(absPath)
			if err != nil {
				return fmt.Errorf("%v: could not dereference path: %v", path, err)
			}
		}

		//TODO PERF no need to retrieve file: we merely need to know it exists
		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			return fmt.Errorf("%v: could not retrieve file: %v", path, err)
		}
		if file == nil {
			action(absPath)
		}

		if recursive {
			entries, err := directoryEntries(path)
			if err != nil {
				return err
			}

			findUntaggedFunc(store, tx, entries, true, followSymlinks, action)
		}
	}

	return nil
}
Exemple #9
0
func listTagsForPaths(store *storage.Storage, tx *storage.Tx, paths []string, showCount, onePerLine, explicitOnly, printPath, colour bool) (error, warnings) {
	warnings := make(warnings, 0, 10)

	printPath = printPath || len(paths) > 1 || !stdoutIsCharDevice()

	for index, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return err, warnings
		}

		log.Infof(2, "%v: retrieving tags.", path)

		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			warnings = append(warnings, err.Error())
			continue
		}

		var tagNames []string
		if file != nil {
			tagNames, err = tagNamesForFile(store, tx, file.Id, explicitOnly, colour)
			if err != nil {
				return err, warnings
			}
		} else {
			_, err := os.Stat(absPath)
			if err != nil {
				switch {
				case os.IsPermission(err):
					warnings = append(warnings, fmt.Sprintf("%v: permission denied", path))
					continue
				case os.IsNotExist(err):
					warnings = append(warnings, fmt.Sprintf("%v: no such file", path))
					continue
				default:
					return fmt.Errorf("%v: could not stat file: %v", path, err), warnings
				}
			}
		}

		switch {
		case showCount:
			if printPath {
				fmt.Print(path + ": ")
			}

			fmt.Println(strconv.Itoa(len(tagNames)))
		case onePerLine:
			if index > 0 {
				fmt.Println()
			}

			if printPath {
				fmt.Println(path + ":")
			}

			for _, tagName := range tagNames {
				fmt.Println(tagName)
			}
		default:
			if printPath {
				fmt.Print(path + ":")

				for _, tagName := range tagNames {
					fmt.Print(" " + tagName)
				}

				fmt.Println()
			} else {
				terminal.PrintColumns(tagNames)
			}
		}
	}

	return nil, warnings
}
Exemple #10
0
func statusPaths(store *storage.Storage, tx *storage.Tx, paths []string, dirOnly, followSymlinks bool) (*StatusReport, error) {
	report := NewReport()

	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return nil, fmt.Errorf("%v: could not get absolute path: %v", path, err)
		}

		log.Infof(2, "%v: resolving file", path)

		resolvedPath := absPath

		stat, err := os.Lstat(absPath)
		if err != nil {
			switch {
			case os.IsNotExist(err), os.IsPermission(err):
				stat = emptyStat{}
			default:
				return nil, fmt.Errorf("%v: could not stat path: %v", path, err)
			}
		} else if stat.Mode()&os.ModeSymlink != 0 {
			resolvedPath, err = _path.Dereference(absPath)
			if err != nil {
				return nil, fmt.Errorf("%v: could not dereference symbolic link: %v", path, err)
			}
		}

		log.Infof(2, "%v: checking file in database", path)

		file, err := store.FileByPath(tx, resolvedPath)
		if err != nil {
			return nil, fmt.Errorf("%v: could not retrieve file: %v", path, err)
		}
		if file != nil {
			err = statusCheckFile(absPath, file, report)
			if err != nil {
				return nil, err
			}
		}

		if !dirOnly && (stat.Mode()&os.ModeSymlink == 0 || followSymlinks) {
			log.Infof(2, "%v: retrieving files from database.", path)

			files, err := store.FilesByDirectory(tx, resolvedPath)
			if err != nil {
				return nil, fmt.Errorf("%v: could not retrieve files for directory: %v", path, err)
			}

			err = statusCheckFiles(files, report)
			if err != nil {
				return nil, err
			}
		}

		err = findNewFiles(absPath, report, dirOnly, followSymlinks)
		if err != nil {
			return nil, err
		}
	}

	return report, nil
}
Exemple #11
0
func tagPath(store *storage.Storage, tx *storage.Tx, path string, pairs []entities.TagIdValueIdPair, explicit, recursive, force bool, fileFingerprintAlg, dirFingerprintAlg, symlinkFingerprintAlg string, reportDuplicates bool) error {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return fmt.Errorf("%v: could not get absolute path: %v", path, err)
	}

	stat, err := os.Stat(path)
	if err != nil {
		switch {
		case os.IsNotExist(err), os.IsPermission(err):
			if !force {
				return err
			} else {
				stat = emptyStat{}
			}
		default:
			return err
		}
	}

	log.Infof(2, "%v: checking if file exists in database", path)

	file, err := store.FileByPath(tx, absPath)
	if err != nil {
		return fmt.Errorf("%v: could not retrieve file: %v", path, err)
	}
	if file == nil {
		log.Infof(2, "%v: creating fingerprint", path)

		fp, err := fingerprint.Create(path, fileFingerprintAlg, dirFingerprintAlg, symlinkFingerprintAlg)
		if err != nil {
			if !force || !(os.IsNotExist(err) || os.IsPermission(err)) {
				return fmt.Errorf("%v: could not create fingerprint: %v", path, err)
			}
		}

		if fp != fingerprint.Empty && reportDuplicates {
			log.Infof(2, "%v: checking for duplicates", path)

			count, err := store.FileCountByFingerprint(tx, fp)
			if err != nil {
				return fmt.Errorf("%v: could not identify duplicates: %v", path, err)
			}
			if count != 0 {
				log.Warnf("'%v' is a duplicate", path)
			}
		}

		log.Infof(2, "%v: adding file", path)

		file, err = store.AddFile(tx, path, fp, stat.ModTime(), int64(stat.Size()), stat.IsDir())
		if err != nil {
			return fmt.Errorf("%v: could not add file to database: %v", path, err)
		}
	}

	if !explicit {
		pairs, err = removeAlreadyAppliedTagValuePairs(store, tx, pairs, file)
		if err != nil {
			return fmt.Errorf("%v: could not remove applied tags: %v", path, err)
		}
	}

	log.Infof(2, "%v: applying tags.", path)

	for _, pair := range pairs {
		if _, err = store.AddFileTag(tx, file.Id, pair.TagId, pair.ValueId); err != nil {
			return fmt.Errorf("%v: could not apply tags: %v", file.Path(), err)
		}
	}

	if recursive && stat.IsDir() {
		if err = tagRecursively(store, tx, path, pairs, explicit, force, fileFingerprintAlg, dirFingerprintAlg, symlinkFingerprintAlg, reportDuplicates); err != nil {
			return err
		}
	}

	return nil
}
Exemple #12
0
func untagPaths(store *storage.Storage, tx *storage.Tx, paths, tagArgs []string, recursive, followSymlinks bool) (error, warnings) {
	warnings := make(warnings, 0, 10)

	files := make(entities.Files, 0, len(paths))
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if err != nil {
			return fmt.Errorf("%v: could not get absolute path: %v", path, err), warnings
		}

		log.Infof(2, "%v: resolving path", path)

		stat, err := os.Lstat(absPath)
		if err != nil {
			switch {
			case os.IsNotExist(err), os.IsPermission(err):
				// ignore
			default:
				return err, nil
			}
		} else if stat.Mode()&os.ModeSymlink != 0 && followSymlinks {
			absPath, err = _path.Dereference(absPath)
			if err != nil {
				return err, nil
			}
		}

		file, err := store.FileByPath(tx, absPath)
		if err != nil {
			return fmt.Errorf("%v: could not retrieve file: %v", path, err), warnings
		}
		if file == nil {
			warnings = append(warnings, fmt.Sprintf("%v: file is not tagged", path))
			continue
		}

		files = append(files, file)

		if recursive {
			childFiles, err := store.FilesByDirectory(tx, file.Path())
			if err != nil {
				return fmt.Errorf("%v: could not retrieve files for directory: %v", file.Path()), warnings
			}

			files = append(files, childFiles...)
		}
	}

	for _, tagArg := range tagArgs {
		tagName, valueName := parseTagEqValueName(tagArg)

		tag, err := store.TagByName(tx, tagName)
		if err != nil {
			return fmt.Errorf("could not retrieve tag '%v': %v", tagName, err), warnings
		}
		if tag == nil {
			warnings = append(warnings, fmt.Sprintf("no such tag '%v'", tagName))
			continue
		}

		value, err := store.ValueByName(tx, valueName)
		if err != nil {
			return fmt.Errorf("could not retrieve value '%v': %v", valueName, err), warnings
		}
		if value == nil {
			warnings = append(warnings, fmt.Sprintf("no such value '%v'", valueName))
			continue
		}

		for _, file := range files {
			if err := store.DeleteFileTag(tx, file.Id, tag.Id, value.Id); err != nil {
				switch err.(type) {
				case storage.FileTagDoesNotExist:
					exists, err := store.FileTagExists(tx, file.Id, tag.Id, value.Id, false)
					if err != nil {
						return fmt.Errorf("could not check if tag exists: %v", err), warnings
					}

					if exists {
						if value.Id != 0 {
							warnings = append(warnings, fmt.Sprintf("%v: cannot remove '%v=%v': delete implication  to remove this tag.", file.Path(), tag.Name, value.Name))
						} else {
							warnings = append(warnings, fmt.Sprintf("%v: cannot remove '%v': delete implication to remove this tag.", file.Path(), tag.Name))
						}
					} else {
						if value.Id != 0 {
							warnings = append(warnings, fmt.Sprintf("%v: file is not tagged '%v=%v'.", file.Path(), tag.Name, value.Name))
						} else {
							warnings = append(warnings, fmt.Sprintf("%v: file is not tagged '%v'.", file.Path(), tag.Name))
						}
					}
				default:
					return fmt.Errorf("%v: could not remove tag '%v', value '%v': %v", file.Path(), tag.Name, value.Name, err), warnings
				}
			}
		}
	}

	return nil, warnings
}
Exemple #13
0
func fullRepair(store *storage.Storage, tx *storage.Tx, searchPaths []string, limitPath string, removeMissing, recalcUnmodified, rationalize, pretend bool) error {
	absLimitPath := ""
	if limitPath != "" {
		var err error
		absLimitPath, err = filepath.Abs(limitPath)
		if err != nil {
			return fmt.Errorf("%v: could not determine absolute path", err)
		}
	}

	settings, err := store.Settings(tx)
	if err != nil {
		return err
	}

	log.Infof(2, "retrieving files under '%v' from the database", absLimitPath)

	dbFiles, err := store.FilesByDirectory(tx, absLimitPath)
	if err != nil {
		return fmt.Errorf("could not retrieve files from storage: %v", err)
	}

	dbFile, err := store.FileByPath(tx, absLimitPath)
	if err != nil {
		return fmt.Errorf("could not retrieve file from storage: %v", err)
	}

	if dbFile != nil {
		dbFiles = append(dbFiles, dbFile)
	}

	log.Infof(2, "retrieved %v files from the database for path '%v'", len(dbFiles), absLimitPath)

	unmodfied, modified, missing := determineStatuses(dbFiles)

	if recalcUnmodified {
		if err = repairUnmodified(store, tx, unmodfied, pretend, settings); err != nil {
			return err
		}
	}

	if err = repairModified(store, tx, modified, pretend, settings); err != nil {
		return err
	}

	if err = repairMoved(store, tx, missing, searchPaths, pretend, settings); err != nil {
		return err
	}

	if err = repairMissing(store, tx, missing, pretend, removeMissing); err != nil {
		return err
	}

	if err = deleteUntaggedFiles(store, tx, dbFiles); err != nil {
		return err
	}

	if rationalize {
		if err = rationalizeFileTags(store, tx, dbFiles); err != nil {
			return err
		}
	}

	return nil
}