func repairModified(store *storage.Storage, modified fileIdAndInfoMap, pretend bool, fingerprintAlgorithm string) error { log.Infof(2, "repairing modified files") for path, fileIdAndStat := range modified { fileId := fileIdAndStat.fileId stat := fileIdAndStat.stat log.Infof(1, "%v: modified", path) fingerprint, err := fingerprint.Create(path, fingerprintAlgorithm) if err != nil { return fmt.Errorf("%v: could not create fingerprint: %v", path, err) } if !pretend { _, err := store.UpdateFile(fileId, path, fingerprint, stat.ModTime(), stat.Size(), stat.IsDir()) if err != nil { return fmt.Errorf("%v: could not update file in database: %v", path, err) } } } return nil }
func statusCheckFile(file *entities.File, report *StatusReport) error { relPath := path.Rel(file.Path()) log.Infof(2, "%v: checking file status.", file.Path()) stat, err := os.Stat(file.Path()) if err != nil { switch { case os.IsNotExist(err): log.Infof(2, "%v: file is missing.", file.Path()) report.AddRow(Row{relPath, MISSING}) return nil case os.IsPermission(err): log.Warnf("%v: permission denied.", file.Path()) case strings.Contains(err.Error(), "not a directory"): report.AddRow(Row{relPath, MISSING}) return nil default: return fmt.Errorf("%v: could not stat: %v", file.Path(), err) } } else { if stat.Size() != file.Size || stat.ModTime().UTC() != file.ModTime { log.Infof(2, "%v: file is modified.", file.Path()) report.AddRow(Row{relPath, MODIFIED}) } else { log.Infof(2, "%v: file is unchanged.", file.Path()) report.AddRow(Row{relPath, TAGGED}) } } return nil }
func (vfs FuseVfs) Rmdir(name string, context *fuse.Context) fuse.Status { log.Infof(2, "BEGIN Rmdir(%v)", name) defer log.Infof(2, "END Rmdir(%v)", name) path := vfs.splitPath(name) switch path[0] { case tagsDir: if len(path) != 2 { // can only remove top-level tag directories return fuse.EPERM } tagName := path[1] tag, err := vfs.store.TagByName(tagName) if err != nil { log.Fatalf("could not retrieve tag '%v': %v", tagName, err) } if tag == nil { return fuse.ENOENT } count, err := vfs.store.FileTagCountByTagId(tag.Id) if err != nil { log.Fatalf("could not retrieve file-tag count for tag '%v': %v", tagName, err) } if count > 0 { return fuse.Status(syscall.ENOTEMPTY) } if err := vfs.store.DeleteTag(tag.Id); err != nil { log.Fatalf("could not delete tag '%v': %v", tagName, err) } if err := vfs.store.Commit(); err != nil { log.Fatalf("could not commit transaction: %v", err) } return fuse.OK case queriesDir: if len(path) != 2 { // can only remove top-level queries directories return fuse.EPERM } text := path[1] if err := vfs.store.DeleteQuery(text); err != nil { log.Fatalf("could not remove tag '%v': %v", name, err) } if err := vfs.store.Commit(); err != nil { log.Fatalf("could not commit transaction: %v", err) } return fuse.OK } return fuse.ENOSYS }
func (vfs FuseVfs) getQueryAttr() (*fuse.Attr, fuse.Status) { log.Infof(2, "BEGIN getQueryAttr") defer log.Infof(2, "END getQueryAttr") now := time.Now() return &fuse.Attr{Mode: fuse.S_IFDIR | 0755, Nlink: 2, Size: 0, Mtime: uint64(now.Unix()), Mtimensec: uint32(now.Nanosecond())}, fuse.OK }
func (vfs FuseVfs) openQueryEntryDir(path []string) ([]fuse.DirEntry, fuse.Status) { log.Infof(2, "BEGIN openQueryEntryDir(%v)", path) defer log.Infof(2, "END openQueryEntryDir(%v)", path) queryText := path[0] expression, err := query.Parse(queryText) if err != nil { return nil, fuse.ENOENT } tagNames := query.TagNames(expression) tags, err := vfs.store.TagsByNames(tagNames) for _, tagName := range tagNames { if !containsTag(tags, tagName) { return nil, fuse.ENOENT } } files, err := vfs.store.QueryFiles(expression) if err != nil { log.Fatalf("could not query files: %v", err) } entries := make([]fuse.DirEntry, 0, len(files)) for _, file := range files { linkName := vfs.getLinkName(file) entries = append(entries, fuse.DirEntry{Name: linkName, Mode: fuse.S_IFLNK}) } return entries, fuse.OK }
func (vfs FuseVfs) Mkdir(name string, mode uint32, context *fuse.Context) fuse.Status { log.Infof(2, "BEGIN Mkdir(%v)", name) defer log.Infof(2, "END Mkdir(%v)", name) path := vfs.splitPath(name) if len(path) != 2 { return fuse.EPERM } switch path[0] { case tagsDir: name := path[1] if _, err := vfs.store.AddTag(name); err != nil { log.Fatalf("could not create tag '%v': %v", name, err) } if err := vfs.store.Commit(); err != nil { log.Fatalf("could not commit transaction: %v", err) } return fuse.OK case queriesDir: return fuse.EINVAL } return fuse.ENOSYS }
func (vfs FuseVfs) topDirectories() ([]fuse.DirEntry, fuse.Status) { log.Infof(2, "BEGIN topDirectories") defer log.Infof(2, "END topDirectories") entries := []fuse.DirEntry{fuse.DirEntry{Name: tagsDir, Mode: fuse.S_IFDIR}, fuse.DirEntry{Name: queriesDir, Mode: fuse.S_IFDIR}} return entries, fuse.OK }
func (vfs FuseVfs) Open(name string, flags uint32, context *fuse.Context) (nodefs.File, fuse.Status) { log.Infof(2, "BEGIN Open(%v)", name) defer log.Infof(2, "END Open(%v)", name) if name == filepath.Join(queriesDir, queryHelpFilename) { return nodefs.NewDataFile([]byte(queryDirHelp)), fuse.OK } return nil, fuse.ENOSYS }
func (vfs FuseVfs) openTaggedEntryDir(path []string) ([]fuse.DirEntry, fuse.Status) { log.Infof(2, "BEGIN openTaggedEntryDir(%v)", path) defer log.Infof(2, "END openTaggedEntryDir(%v)", path) expression := query.HasAll(path) files, err := vfs.store.QueryFiles(expression) if err != nil { log.Fatalf("could not query files: %v", err) } tagNames := make(map[string]interface{}, len(path)) for _, tagName := range path { tagNames[tagName] = nil } furtherTagNames := make([]string, 0, 10) for _, file := range files { fileTags, err := vfs.store.FileTagsByFileId(file.Id) if err != nil { log.Fatalf("could not retrieve file-tags for file '%v': %v", file.Id, err) } tagIds := make([]uint, len(fileTags)) for index, fileTag := range fileTags { tagIds[index] = fileTag.TagId } tags, err := vfs.store.TagsByIds(tagIds) if err != nil { log.Fatalf("could not retrieve tags: %v", err) } for _, tag := range tags { _, has := tagNames[tag.Name] if !has { if !containsName(furtherTagNames, tag.Name) { furtherTagNames = append(furtherTagNames, tag.Name) } } } } entries := make([]fuse.DirEntry, 0, len(files)+len(furtherTagNames)) for _, tagName := range furtherTagNames { entries = append(entries, fuse.DirEntry{Name: tagName, Mode: fuse.S_IFDIR | 0755}) } for _, file := range files { linkName := vfs.getLinkName(file) entries = append(entries, fuse.DirEntry{Name: linkName, Mode: fuse.S_IFLNK}) } return entries, fuse.OK }
func (vfs FuseVfs) Readlink(name string, context *fuse.Context) (string, fuse.Status) { log.Infof(2, "BEGIN Readlink(%v)", name) defer log.Infof(2, "END Readlink(%v)", name) path := vfs.splitPath(name) switch path[0] { case tagsDir, queriesDir: return vfs.readTaggedEntryLink(path[1:]) } return "", fuse.ENOENT }
func (vfs FuseVfs) getTagsAttr() (*fuse.Attr, fuse.Status) { log.Infof(2, "BEGIN getTagsAttr") defer log.Infof(2, "END getTagsAttr") tagCount, err := vfs.store.TagCount() if err != nil { log.Fatalf("could not get tag count: %v", err) } now := time.Now() return &fuse.Attr{Mode: fuse.S_IFDIR | 0755, Nlink: 2, Size: uint64(tagCount), Mtime: uint64(now.Unix()), Mtimensec: uint32(now.Nanosecond())}, fuse.OK }
func (vfs FuseVfs) getQueryEntryAttr(path []string) (*fuse.Attr, fuse.Status) { log.Infof(2, "BEGIN getQueryEntryAttr(%v)", path) defer log.Infof(2, "END getQueryEntryAttr(%v)", path) pathLength := len(path) name := path[pathLength-1] if len(path) == 1 && path[0] == queryHelpFilename { now := time.Now() return &fuse.Attr{Mode: fuse.S_IFREG | 0444, Nlink: 1, Size: uint64(len(queryDirHelp)), Mtime: uint64(now.Unix()), Mtimensec: uint32(now.Nanosecond())}, fuse.OK } if len(path) > 1 { fileId := vfs.parseFileId(name) if fileId != 0 { return vfs.getFileEntryAttr(fileId) } return nil, fuse.ENOENT } queryText := path[0] if queryText[len(queryText)-1] == ' ' { // prevent multiple entries for same query when typing path in a GUI return nil, fuse.ENOENT } expression, err := query.Parse(queryText) if err != nil { return nil, fuse.ENOENT } tagNames := query.TagNames(expression) tags, err := vfs.store.TagsByNames(tagNames) for _, tagName := range tagNames { if !containsTag(tags, tagName) { return nil, fuse.ENOENT } } _, _ = vfs.store.AddQuery(queryText) if err := vfs.store.Commit(); err != nil { log.Fatalf("could not commit transaction: %v", err) } now := time.Now() return &fuse.Attr{Mode: fuse.S_IFDIR | 0755, Nlink: 2, Size: uint64(0), Mtime: uint64(now.Unix()), Mtimensec: uint32(now.Nanosecond())}, fuse.OK }
func repairMissing(store *storage.Storage, missing databaseFileMap, pretend, force bool) error { for path, dbFile := range missing { if force && !pretend { if err := store.DeleteFileTagsByFileId(dbFile.Id); err != nil { return fmt.Errorf("%v: could not delete file-tags: %v", path, err) } log.Infof(1, "%v: removed", path) } else { log.Infof(1, "%v: missing", path) } } return nil }
func (vfs FuseVfs) Unlink(name string, context *fuse.Context) fuse.Status { log.Infof(2, "BEGIN Unlink(%v)", name) defer log.Infof(2, "END Unlink(%v)", name) fileId := vfs.parseFileId(name) if fileId == 0 { // can only unlink file symbolic links return fuse.EPERM } file, err := vfs.store.File(fileId) if err != nil { log.Fatal("could not retrieve file '%v': %v", fileId, err) } if file == nil { // reply ok if file doesn't exist otherwise recursive deletes fail return fuse.OK } path := vfs.splitPath(name) switch path[0] { case tagsDir: tagName := path[len(path)-2] //TODO value name tag, err := vfs.store.TagByName(tagName) if err != nil { log.Fatal(err) } if tag == nil { log.Fatalf("could not retrieve tag '%v'.", tagName) } if err = vfs.store.DeleteFileTag(fileId, tag.Id, 0); err != nil { log.Fatal(err) } if err := vfs.store.Commit(); err != nil { log.Fatalf("could not commit transaction: %v", err) } return fuse.OK case queriesDir: return fuse.EPERM } return fuse.ENOSYS }
func listValuesForTag(store *storage.Storage, tagName string, showCount, onePerLine bool) error { tag, err := store.TagByName(tagName) if err != nil { return fmt.Errorf("could not retrieve tag '%v': %v", tagName, err) } if tag == nil { return fmt.Errorf("no such tag, '%v'.", tagName) } log.Infof(2, "retrieving values for tag '%v'.", tagName) values, err := store.ValuesByTag(tag.Id) if err != nil { return fmt.Errorf("could not retrieve values for tag '%v': %v", tagName, err) } if showCount { fmt.Println(len(values)) } else { if onePerLine { for _, value := range values { fmt.Println(value.Name) } } else { valueNames := make([]string, len(values)) for index, value := range values { valueNames[index] = value.Name } format.Columns(valueNames, terminalWidth()) } } return nil }
// Opens the database at the specified path func OpenAt(path string) (*Database, error) { log.Infof(2, "opening database at '%v'.", path) _, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { log.Warnf("creating database at '%v'.", path) } else { log.Warnf("could not stat database: %v", err) } } connection, err := sql.Open("sqlite3", path) if err != nil { return nil, fmt.Errorf("could not open database: %v", err) } transaction, err := connection.Begin() if err != nil { return nil, fmt.Errorf("could not begin transaciton: %v", err) } database := &Database{connection, transaction} if err := database.CreateSchema(); err != nil { return nil, errors.New("could not create database schema: " + err.Error()) } return database, nil }
func unmount(path string) error { log.Info(2, "searching path for fusermount.") fusermountPath, err := exec.LookPath("fusermount") if err != nil { return fmt.Errorf("could not find 'fusermount': ensure fuse is installed: %v", err) } log.Infof(2, "running: %v -u %v.", fusermountPath, path) process, err := os.StartProcess(fusermountPath, []string{fusermountPath, "-u", path}, &os.ProcAttr{}) if err != nil { return fmt.Errorf("could not start 'fusermount': %v", err) } log.Info(2, "waiting for process to exit.") processState, err := process.Wait() if err != nil { return fmt.Errorf("error waiting for process to exit: %v", err) } if !processState.Success() { return fmt.Errorf("could not unmount virtual filesystem.") } return nil }
func determineStatuses(fsPaths fileInfoMap, dbPaths databaseFileMap) (tagged databaseFileMap, untagged fileInfoMap, modified fileIdAndInfoMap, missing databaseFileMap) { log.Infof(2, "determining file statuses") tagged = make(databaseFileMap, 100) untagged = make(fileInfoMap, 100) modified = make(fileIdAndInfoMap, 100) missing = make(databaseFileMap, 100) for path, stat := range fsPaths { if dbFile, isTagged := dbPaths[path]; isTagged { if dbFile.ModTime == stat.ModTime().UTC() && dbFile.Size == stat.Size() { tagged[path] = dbFile } else { modified[path] = struct { fileId uint stat os.FileInfo }{dbFile.Id, stat} } } else { untagged[path] = stat } } for path, dbFile := range dbPaths { if _, found := fsPaths[path]; !found { missing[path] = dbFile } } return tagged, untagged, modified, missing }
func findDuplicatesInDb() error { store, err := storage.Open() if err != nil { return fmt.Errorf("could not open storage: %v", err) } defer store.Close() log.Info(2, "identifying duplicate files.") fileSets, err := store.DuplicateFiles() if err != nil { return fmt.Errorf("could not identify duplicate files: %v", err) } log.Infof(2, "found %v sets of duplicate files.", len(fileSets)) for index, fileSet := range fileSets { if index > 0 { fmt.Println() } fmt.Printf("Set of %v duplicates:\n", len(fileSet)) for _, file := range fileSet { relPath := _path.Rel(file.Path()) fmt.Printf(" %v\n", relPath) } } return nil }
func (vfs FuseVfs) tagDirectories() ([]fuse.DirEntry, fuse.Status) { log.Infof(2, "BEGIN tagDirectories") defer log.Infof(2, "END tagDirectories") tags, err := vfs.store.Tags() if err != nil { log.Fatalf("Could not retrieve tags: %v", err) } entries := make([]fuse.DirEntry, len(tags)) for index, tag := range tags { entries[index] = fuse.DirEntry{Name: tag.Name, Mode: fuse.S_IFDIR} } return entries, fuse.OK }
func tagPath(store *storage.Storage, path string, tagValuePairs []TagValuePair, recursive bool, fingerprintAlgorithm string) error { absPath, err := filepath.Abs(path) if err != nil { return fmt.Errorf("%v: could not get absolute path: %v", path, err) } stat, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { stat, err = os.Lstat(path) if err != nil { return err } } else { return err } } log.Infof(2, "%v: checking if file exists", absPath) file, err := store.FileByPath(absPath) if err != nil { return fmt.Errorf("%v: could not retrieve file: %v", path, err) } if file == nil { file, err = addFile(store, absPath, stat.ModTime(), uint(stat.Size()), stat.IsDir(), fingerprintAlgorithm) if err != nil { return fmt.Errorf("%v: could not add file: %v", path, err) } } log.Infof(2, "%v: applying tags.", file.Path()) for _, tagValuePair := range tagValuePairs { if _, err = store.AddFileTag(file.Id, tagValuePair.TagId, tagValuePair.ValueId); err != nil { return fmt.Errorf("%v: could not apply tags: %v", file.Path(), err) } } if recursive && stat.IsDir() { if err = tagRecursively(store, path, tagValuePairs, fingerprintAlgorithm); err != nil { return err } } return nil }
func addFile(store *storage.Storage, path string, modTime time.Time, size uint, isDir bool, fingerprintAlgorithm string) (*entities.File, error) { log.Infof(2, "%v: creating fingerprint", path) fingerprint, err := fingerprint.Create(path, fingerprintAlgorithm) if err != nil { return nil, fmt.Errorf("%v: could not create fingerprint: %v", path, err) } log.Infof(2, "%v: adding file.", path) file, err := store.AddFile(path, fingerprint, modTime, int64(size), isDir) if err != nil { return nil, fmt.Errorf("%v: could not add file to database: %v", path, err) } return file, nil }
func (vfs FuseVfs) readTaggedEntryLink(path []string) (string, fuse.Status) { log.Infof(2, "BEGIN readTaggedEntryLink(%v)", path) defer log.Infof(2, "END readTaggedEntryLink(%v)", path) name := path[len(path)-1] fileId := vfs.parseFileId(name) if fileId == 0 { return "", fuse.ENOENT } file, err := vfs.store.File(fileId) if err != nil { log.Fatalf("could not find file %v in database.", fileId) } return file.Path(), fuse.OK }
// Executes a SQL query. func (db *Database) Exec(sql string, args ...interface{}) (sql.Result, error) { if log.Verbosity >= 3 { log.Infof(3, "executing update\n"+sql) for index, arg := range args { log.Info(3, "Arg %v = %v", index, arg) } } return db.transaction.Exec(sql, args...) }
// Executes a SQL query returning rows. func (db *Database) ExecQuery(sql string, args ...interface{}) (*sql.Rows, error) { if log.Verbosity >= 3 { log.Infof(3, "executing query\n"+sql) for index, arg := range args { log.Info(3, "Arg %v = %v", index, arg) } } return db.transaction.Query(sql, args...) }
func (vfs FuseVfs) queriesDirectories() ([]fuse.DirEntry, fuse.Status) { log.Infof(2, "BEGIN queriesDirectories") defer log.Infof(2, "END queriesDirectories") queries, err := vfs.store.Queries() if err != nil { log.Fatalf("could not retrieve queries: %v", err) } if len(queries) == 0 { return []fuse.DirEntry{fuse.DirEntry{Name: queryHelpFilename, Mode: fuse.S_IFREG}}, fuse.OK } entries := make([]fuse.DirEntry, len(queries)) for index, query := range queries { entries[index] = fuse.DirEntry{Name: query.Text, Mode: fuse.S_IFDIR} } return entries, fuse.OK }
func listTagsForWorkingDirectory(store *storage.Storage, showCount, onePerLine bool) error { file, err := os.Open(".") if err != nil { return fmt.Errorf("could not open working directory: %v", err) } defer file.Close() dirNames, err := file.Readdirnames(0) if err != nil { return fmt.Errorf("could not list working directory contents: %v", err) } sort.Strings(dirNames) for _, dirName := range dirNames { log.Infof(2, "%v: retrieving tags.", dirName) file, err := store.FileByPath(dirName) if err != nil { log.Warn(err.Error()) continue } if file == nil { continue } fileTags, err := store.FileTagsByFileId(file.Id) if err != nil { return fmt.Errorf("could not retrieve file-tags: %v", err) } tagNames, err := lookupTagNames(store, fileTags) if err != nil { return err } if showCount { fmt.Println(dirName + ": " + strconv.Itoa(len(tagNames))) } else { if onePerLine { fmt.Println(dirName) for _, tagName := range tagNames { fmt.Println(tagName) } fmt.Println() } else { fmt.Println(dirName + ": " + strings.Join(tagNames, " ")) } } } return nil }
func repairMoved(store *storage.Storage, missing databaseFileMap, untagged fileInfoMap, pretend bool, fingerprintAlgorithm string) error { log.Infof(2, "repairing moved files") moved := make([]string, 0, 10) for path, dbFile := range missing { log.Infof(2, "%v: searching for new location", path) for candidatePath, stat := range untagged { if stat.Size() == dbFile.Size { fingerprint, err := fingerprint.Create(candidatePath, fingerprintAlgorithm) if err != nil { return fmt.Errorf("%v: could not create fingerprint: %v", path, err) } if fingerprint == dbFile.Fingerprint { log.Infof(1, "%v: moved to %v", path, candidatePath) moved = append(moved, path) if !pretend { _, err := store.UpdateFile(dbFile.Id, candidatePath, dbFile.Fingerprint, stat.ModTime(), dbFile.Size, dbFile.IsDir) if err != nil { return fmt.Errorf("%v: could not update file in database: %v", path, err) } } delete(untagged, candidatePath) break } } } } for _, path := range moved { delete(missing, path) } return nil }
func untagPathsAll(paths []string, recursive bool) error { store, err := storage.Open() if err != nil { return fmt.Errorf("could not open storage: %v", err) } defer store.Close() defer store.Commit() wereErrors := false for _, path := range paths { absPath, err := filepath.Abs(path) if err != nil { return fmt.Errorf("%v: could not get absolute path: %v", path, err) } file, err := store.FileByPath(absPath) if err != nil { return fmt.Errorf("%v: could not retrieve file: %v", path, err) } if file == nil { log.Warnf("%v: file is not tagged.", path) wereErrors = true continue } log.Infof(2, "%v: removing all tags.", file.Path()) if err := store.DeleteFileTagsByFileId(file.Id); err != nil { return fmt.Errorf("%v: could not remove file's tags: %v", file.Path(), err) } if recursive { childFiles, err := store.FilesByDirectory(file.Path()) if err != nil { return fmt.Errorf("%v: could not retrieve files for directory: %v", file.Path()) } for _, childFile := range childFiles { if err := store.DeleteFileTagsByFileId(childFile.Id); err != nil { return fmt.Errorf("%v: could not remove file's tags: %v", childFile.Path(), err) } } } } if wereErrors { return blankError } return nil }
func (vfs FuseVfs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry, fuse.Status) { log.Infof(2, "BEGIN OpenDir(%v)", name) defer log.Infof(2, "END OpenDir(%v)", name) switch name { case "": return vfs.topDirectories() case tagsDir: return vfs.tagDirectories() case queriesDir: return vfs.queriesDirectories() } path := vfs.splitPath(name) switch path[0] { case tagsDir: return vfs.openTaggedEntryDir(path[1:]) case queriesDir: return vfs.openQueryEntryDir(path[1:]) } return nil, fuse.ENOENT }