func TestDupesNoneUntaggedFile(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
path := filepath.Join(os.TempDir(), "tmsu-file")
_, err = os.Create(path)
if err != nil {
test.Fatal(err)
}
defer os.Remove(path)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
_, err = store.AddFile("/tmp/a", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/a/b", fingerprint.Fingerprint("def"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/b", fingerprint.Fingerprint("ghi"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/e/f", fingerprint.Fingerprint("klm"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/a/d", fingerprint.Fingerprint("nop"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := DupesCommand.Exec(Options{}, []string{path}); err != nil {
test.Fatal(err)
}
// validate
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "", string(bytes))
}
func TestDupesMultipleUntaggedFile(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
path := filepath.Join(os.TempDir(), "tmsu-file")
_, err = os.Create(path)
if err != nil {
test.Fatal(err)
}
defer os.Remove(path)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
_, err = store.AddFile("/tmp/a", fingerprint.Fingerprint("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/a/b", fingerprint.Fingerprint("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/b", fingerprint.Fingerprint("xxx"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/e/f", fingerprint.Fingerprint("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/a/d", fingerprint.Fingerprint("xxx"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := DupesCommand.Exec(Options{}, []string{path}); err != nil {
test.Fatal(err)
}
// validate
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "/tmp/a\n/tmp/a/b\n/tmp/e/f\n", string(bytes))
}
func TestCopySuccessful(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
fileA, err := store.AddFile("/tmp/a", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
fileAB, err := store.AddFile("/tmp/a/b", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
sourceTag, err := store.AddTag("source")
if err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileA.Id, sourceTag.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileAB.Id, sourceTag.Id, 0); err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := CopyCommand.Exec(Options{}, []string{"source", "dest"}); err != nil {
test.Fatal(err)
}
// validate
destTag, err := store.TagByName("dest")
if err != nil {
test.Fatal(err)
}
if destTag == nil {
test.Fatal("Destination tag does not exist.")
}
expectTags(test, store, fileA, sourceTag, destTag)
expectTags(test, store, fileAB, sourceTag, destTag)
}
func TestDupesMultiple(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
_, err = store.AddFile("/tmp/a", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/a/b", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/b", fingerprint.Fingerprint("def"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/e/f", fingerprint.Fingerprint("def"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/a/d", fingerprint.Fingerprint("def"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := DupesCommand.Exec(Options{}, []string{}); err != nil {
test.Fatal(err)
}
// validate
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "Set of 2 duplicates:\n /tmp/a\n /tmp/a/b\n\nSet of 3 duplicates:\n /tmp/a/d\n /tmp/b\n /tmp/e/f\n", string(bytes))
}
func TestFilesAll(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
_, err = store.AddFile("/tmp/d", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/b/a", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFile("/tmp/b", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := FilesCommand.Exec(Options{Option{"-a", "--all", "", false, ""}}, []string{}); err != nil {
test.Fatal(err)
}
// validate
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "/tmp/b\n/tmp/b/a\n/tmp/d\n", string(bytes))
}
func TestTagsForSingleFile(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
file, err := store.AddFile("/tmp/tmsu/a", fingerprint.Fingerprint("123"), time.Now(), 0, false)
if err != nil {
test.Fatal(err)
}
appleTag, err := store.AddTag("apple")
if err != nil {
test.Fatal(err)
}
bananaTag, err := store.AddTag("banana")
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, appleTag.Id, 0)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, bananaTag.Id, 0)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := TagsCommand.Exec(Options{}, []string{"/tmp/tmsu/a"}); err != nil {
test.Fatal(err)
}
// verify
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "apple\nbanana\n", string(bytes))
}
func TestSingleUntag(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
file, err := store.AddFile("/tmp/tmsu/a", fingerprint.Fingerprint("abc123"), time.Now(), 0, false)
if err != nil {
test.Fatal(err)
}
appleTag, err := store.AddTag("apple")
if err != nil {
test.Fatal(err)
}
bananaTag, err := store.AddTag("banana")
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, appleTag.Id, 0)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, bananaTag.Id, 0)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := UntagCommand.Exec(Options{}, []string{"/tmp/tmsu/a", "apple"}); err != nil {
test.Fatal(err)
}
// validate
fileTags, err := store.FileTags()
if err != nil {
test.Fatal(err)
}
if len(fileTags) != 1 {
test.Fatalf("Expected one file-tag but are %v", len(fileTags))
}
if fileTags[0].TagId != bananaTag.Id {
test.Fatalf("Incorrect tag was applied.")
}
}
// Retrieves the sets of duplicate files within the database.
func (db *Database) DuplicateFiles() ([]entities.Files, error) {
sql := `SELECT id, directory, name, fingerprint, mod_time, size, is_dir
FROM file
WHERE fingerprint IN (
SELECT fingerprint
FROM file
WHERE fingerprint != ''
GROUP BY fingerprint
HAVING count(1) > 1
)
ORDER BY fingerprint, directory || '/' || name`
rows, err := db.ExecQuery(sql)
if err != nil {
return nil, err
}
defer rows.Close()
fileSets := make([]entities.Files, 0, 10)
var fileSet entities.Files
var previousFingerprint fingerprint.Fingerprint
for rows.Next() {
if rows.Err() != nil {
return nil, err
}
var fileId uint
var directory, name, fp string
var modTime time.Time
var size int64
var isDir bool
err = rows.Scan(&fileId, &directory, &name, &fp, &modTime, &size, &isDir)
if err != nil {
return nil, err
}
fingerprint := fingerprint.Fingerprint(fp)
if fingerprint != previousFingerprint {
if fileSet != nil {
fileSets = append(fileSets, fileSet)
}
fileSet = make(entities.Files, 0, 10)
previousFingerprint = fingerprint
}
fileSet = append(fileSet, &entities.File{fileId, directory, name, fingerprint, modTime, size, isDir})
}
// ensure last file set is added
if len(fileSet) > 0 {
fileSets = append(fileSets, fileSet)
}
return fileSets, nil
}
func readFile(rows *sql.Rows) (*entities.File, error) {
if !rows.Next() {
return nil, nil
}
if rows.Err() != nil {
return nil, rows.Err()
}
var fileId uint
var directory, name, fp string
var modTime time.Time
var size int64
var isDir bool
err := rows.Scan(&fileId, &directory, &name, &fp, &modTime, &size, &isDir)
if err != nil {
return nil, err
}
return &entities.File{fileId, directory, name, fingerprint.Fingerprint(fp), modTime, size, isDir}, nil
}
func TestFilesSingleTag(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
fileD, err := store.AddFile("/tmp/d", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
fileBA, err := store.AddFile("/tmp/b/a", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
fileB, err := store.AddFile("/tmp/b", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
tagD, err := store.AddTag("d")
if err != nil {
test.Fatal(err)
}
tagB, err := store.AddTag("b")
if err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileD.Id, tagD.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileB.Id, tagB.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileBA.Id, tagB.Id, 0); err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := FilesCommand.Exec(Options{}, []string{"b"}); err != nil {
test.Fatal(err)
}
// validate
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "/tmp/b\n/tmp/b/a\n", string(bytes))
}
func TestDeleteSuccessful(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
fileD, err := store.AddFile("/tmp/d", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
fileF, err := store.AddFile("/tmp/f", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
fileB, err := store.AddFile("/tmp/b", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
tagDeathrow, err := store.AddTag("deathrow")
if err != nil {
test.Fatal(err)
}
tagFreeman, err := store.AddTag("freeman")
if err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileD.Id, tagDeathrow.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileF.Id, tagFreeman.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileB.Id, tagDeathrow.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileB.Id, tagFreeman.Id, 0); err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := DeleteCommand.Exec(Options{}, []string{"deathrow"}); err != nil {
test.Fatal(err)
}
// validate
tagDeathrow, err = store.TagByName("deathrow")
if err != nil {
test.Fatal(err)
}
if tagDeathrow != nil {
test.Fatal("Deleted tag still exists.")
}
fileTagsD, err := store.FileTagsByFileId(fileD.Id)
if err != nil {
test.Fatal(err)
}
if len(fileTagsD) != 0 {
test.Fatal("Expected no file-tags for file 'd'.")
}
fileTagsF, err := store.FileTagsByFileId(fileF.Id)
if err != nil {
test.Fatal(err)
}
if len(fileTagsF) != 1 {
test.Fatal("Expected one file-tag for file 'f'.")
}
if fileTagsF[0].TagId != tagFreeman.Id {
test.Fatal("Expected file-tag for tag 'freeman'.")
}
fileTagsB, err := store.FileTagsByFileId(fileB.Id)
if err != nil {
test.Fatal(err)
}
if len(fileTagsB) != 1 {
test.Fatal("Expected one file-tag for file 'b'.")
}
if fileTagsB[0].TagId != tagFreeman.Id {
test.Fatal("Expected file-tag for tag 'freeman'.")
}
}
func TestMergeSingleTag(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
fileA, err := store.AddFile("/tmp/a", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
fileA1, err := store.AddFile("/tmp/a/1", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
fileB, err := store.AddFile("/tmp/b", fingerprint.Fingerprint("abc"), time.Now(), 123, true)
if err != nil {
test.Fatal(err)
}
fileB1, err := store.AddFile("/tmp/b/1", fingerprint.Fingerprint("abc"), time.Now(), 123, false)
if err != nil {
test.Fatal(err)
}
tagA, err := store.AddTag("a")
if err != nil {
test.Fatal(err)
}
tagB, err := store.AddTag("b")
if err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileA.Id, tagA.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileA1.Id, tagA.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileB.Id, tagB.Id, 0); err != nil {
test.Fatal(err)
}
if _, err := store.AddFileTag(fileB1.Id, tagB.Id, 0); err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := MergeCommand.Exec(Options{}, []string{"a", "b"}); err != nil {
test.Fatal(err)
}
// validate
tagA, err = store.TagByName("a")
if err != nil {
test.Fatal(err)
}
if tagA != nil {
test.Fatal("Tag 'a' still exists.")
}
tagB, err = store.TagByName("b")
if err != nil {
test.Fatal(err)
}
if tagB == nil {
test.Fatal("Tag 'b' does not exist.")
}
expectTags(test, store, fileA, tagB)
expectTags(test, store, fileA1, tagB)
expectTags(test, store, fileB, tagB)
expectTags(test, store, fileB1, tagB)
}
func TestValuesForMulitpleTags(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
err := redirectStreams()
if err != nil {
test.Fatal(err)
}
defer restoreStreams()
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
file, err := store.AddFile("/tmp/tmsu/a", fingerprint.Fingerprint("123"), time.Now(), 0, false)
if err != nil {
test.Fatal(err)
}
materialTag, err := store.AddTag("material")
if err != nil {
test.Fatal(err)
}
shapeTag, err := store.AddTag("shape")
if err != nil {
test.Fatal(err)
}
woodValue, err := store.AddValue("wood")
if err != nil {
test.Fatal(err)
}
metalValue, err := store.AddValue("metal")
if err != nil {
test.Fatal(err)
}
torroidValue, err := store.AddValue("torroid")
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, materialTag.Id, woodValue.Id)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, materialTag.Id, metalValue.Id)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(file.Id, shapeTag.Id, torroidValue.Id)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := ValuesCommand.Exec(Options{}, []string{"material", "shape"}); err != nil {
test.Fatal(err)
}
// verify
outFile.Seek(0, 0)
bytes, err := ioutil.ReadAll(outFile)
compareOutput(test, "material: metal wood\nshape: torroid\n", string(bytes))
}
func TestUntagAll(test *testing.T) {
// set-up
databasePath := testDatabase()
defer os.Remove(databasePath)
store, err := storage.Open()
if err != nil {
test.Fatal(err)
}
defer store.Close()
fileA, err := store.AddFile("/tmp/tmsu/a", fingerprint.Fingerprint("abc123"), time.Now(), 0, false)
if err != nil {
test.Fatal(err)
}
fileB, err := store.AddFile("/tmp/tmsu/b", fingerprint.Fingerprint("abc123"), time.Now(), 0, false)
if err != nil {
test.Fatal(err)
}
appleTag, err := store.AddTag("apple")
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(fileA.Id, appleTag.Id, 0)
if err != nil {
test.Fatal(err)
}
_, err = store.AddFileTag(fileB.Id, appleTag.Id, 0)
if err != nil {
test.Fatal(err)
}
store.Commit()
// test
if err := UntagCommand.Exec(Options{Option{"--all", "-a", "", false, ""}}, []string{"/tmp/tmsu/a", "/tmp/tmsu/b"}); err != nil {
test.Fatal(err)
}
// validate
fileTags, err := store.FileTags()
if err != nil {
test.Fatal(err)
}
if len(fileTags) != 0 {
test.Fatalf("Expected no file-tags but are %v", len(fileTags))
}
files, err := store.Files()
if err != nil {
test.Fatal(err)
}
if len(files) != 0 {
test.Fatalf("Expected no files but are %v", len(files))
}
}
func findDuplicatesOf(paths []string, recursive bool) error {
store, err := storage.Open()
if err != nil {
return fmt.Errorf("could not open storage: %v", err)
}
defer store.Close()
fingerprintAlgorithmSetting, err := store.Setting("fingerprintAlgorithm")
if err != nil {
return fmt.Errorf("could not retrieve fingerprint algorithm: %v", err)
}
wereErrors := false
for _, path := range paths {
_, err := os.Stat(path)
if err != nil {
switch {
case os.IsNotExist(err):
log.Warnf("%v: no such file", path)
wereErrors = true
continue
case os.IsPermission(err):
log.Warnf("%v: permission denied", path)
wereErrors = true
continue
default:
return err
}
}
}
if wereErrors {
return blankError
}
if recursive {
p, err := _path.Enumerate(paths)
if err != nil {
return fmt.Errorf("could not enumerate paths: %v", err)
}
paths = make([]string, len(p))
for index, path := range p {
paths[index] = path.Path
}
}
first := true
for _, path := range paths {
log.Infof(2, "%v: identifying duplicate files.", path)
fp, err := fingerprint.Create(path, fingerprintAlgorithmSetting.Value)
if err != nil {
return fmt.Errorf("%v: could not create fingerprint: %v", path, err)
}
if fp == fingerprint.Fingerprint("") {
continue
}
files, err := store.FilesByFingerprint(fp)
if err != nil {
return fmt.Errorf("%v: could not retrieve files matching fingerprint '%v': %v", path, fp, err)
}
absPath, err := filepath.Abs(path)
if err != nil {
return fmt.Errorf("%v: could not determine absolute path: %v", path, err)
}
// filter out the file we're searching on
dupes := files.Where(func(file *entities.File) bool { return file.Path() != absPath })
if len(paths) > 1 && len(dupes) > 0 {
if first {
first = false
} else {
fmt.Println()
}
fmt.Printf("%v:\n", path)
for _, dupe := range dupes {
relPath := _path.Rel(dupe.Path())
fmt.Printf(" %v\n", relPath)
}
} else {
for _, dupe := range dupes {
relPath := _path.Rel(dupe.Path())
fmt.Println(relPath)
}
}
}
return nil
}
