// cleanConfigDirectory removes old, unused configuration and index formats, a
// suitable time after they have gone out of fashion.
func cleanConfigDirectory() {
patterns := map[string]time.Duration{
"panic-*.log": 7 * 24 * time.Hour, // keep panic logs for a week
"audit-*.log": 7 * 24 * time.Hour, // keep audit logs for a week
"index": 14 * 24 * time.Hour, // keep old index format for two weeks
"config.xml.v*": 30 * 24 * time.Hour, // old config versions for a month
"*.idx.gz": 30 * 24 * time.Hour, // these should for sure no longer exist
"backup-of-v0.8": 30 * 24 * time.Hour, // these neither
}
for pat, dur := range patterns {
pat = filepath.Join(baseDirs["config"], pat)
files, err := osutil.Glob(pat)
if err != nil {
l.Infoln("Cleaning:", err)
continue
}
for _, file := range files {
info, err := osutil.Lstat(file)
if err != nil {
l.Infoln("Cleaning:", err)
continue
}
if time.Since(info.ModTime()) > dur {
if err = os.RemoveAll(file); err != nil {
l.Infoln("Cleaning:", err)
} else {
l.Infoln("Cleaned away old file", filepath.Base(file))
}
}
}
}
}
func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
qs := r.URL.Query()
current := qs.Get("current")
if current == "" {
if roots, err := osutil.GetFilesystemRoots(); err == nil {
sendJSON(w, roots)
} else {
http.Error(w, err.Error(), 500)
}
return
}
search, _ := osutil.ExpandTilde(current)
pathSeparator := string(os.PathSeparator)
if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
search = search + pathSeparator
}
subdirectories, _ := osutil.Glob(search + "*")
ret := make([]string, 0, len(subdirectories))
for _, subdirectory := range subdirectories {
info, err := os.Stat(subdirectory)
if err == nil && info.IsDir() {
ret = append(ret, subdirectory+pathSeparator)
}
}
sendJSON(w, ret)
}
func (p *rwFolder) moveForConflict(name string) error {
if p.maxConflicts == 0 {
if err := osutil.Remove(name); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
ext := filepath.Ext(name)
withoutExt := name[:len(name)-len(ext)]
newName := withoutExt + time.Now().Format(".sync-conflict-20060102-150405") + ext
err := os.Rename(name, newName)
if os.IsNotExist(err) {
// We were supposed to move a file away but it does not exist. Either
// the user has already moved it away, or the conflict was between a
// remote modification and a local delete. In either way it does not
// matter, go ahead as if the move succeeded.
err = nil
}
if p.maxConflicts > -1 {
matches, gerr := osutil.Glob(withoutExt + ".sync-conflict-????????-??????" + ext)
if gerr == nil && len(matches) > p.maxConflicts {
sort.Sort(sort.Reverse(sort.StringSlice(matches)))
for _, match := range matches[p.maxConflicts:] {
gerr = osutil.Remove(match)
if gerr != nil {
l.Debugln(p, "removing extra conflict", gerr)
}
}
} else if gerr != nil {
l.Debugln(p, "globbing for conflicts", gerr)
}
}
return err
}
func TestGlob(t *testing.T) {
testcases := []string{
`C:\*`,
`\\?\C:\*`,
`\\?\C:\Users`,
`\\?\\\?\C:\Users`,
}
for _, tc := range testcases {
if _, err := osutil.Glob(tc); err != nil {
t.Fatalf("pattern %s failed: %v", tc, err)
}
}
}
func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
qs := r.URL.Query()
current := qs.Get("current")
search, _ := osutil.ExpandTilde(current)
pathSeparator := string(os.PathSeparator)
if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
search = search + pathSeparator
}
subdirectories, _ := osutil.Glob(search + "*")
ret := make([]string, 0, 10)
for _, subdirectory := range subdirectories {
info, err := os.Stat(subdirectory)
if err == nil && info.IsDir() {
ret = append(ret, subdirectory+pathSeparator)
if len(ret) > 9 {
break
}
}
}
sendJSON(w, ret)
}
// rm -rf
func removeAll(dirs ...string) error {
for _, dir := range dirs {
files, err := osutil.Glob(dir)
if err != nil {
return err
}
for _, file := range files {
// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&0700 != 0700 {
os.Chmod(path, 0777)
}
return nil
})
os.RemoveAll(file)
}
}
return nil
}
func (s *apiSvc) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
qs := r.URL.Query()
current := qs.Get("current")
search, _ := osutil.ExpandTilde(current)
pathSeparator := string(os.PathSeparator)
if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
search = search + pathSeparator
}
subdirectories, _ := osutil.Glob(search + "*")
ret := make([]string, 0, 10)
for _, subdirectory := range subdirectories {
info, err := os.Stat(subdirectory)
if err == nil && info.IsDir() {
ret = append(ret, subdirectory+pathSeparator)
if len(ret) > 9 {
break
}
}
}
json.NewEncoder(w).Encode(ret)
}
func TestCLIReset(t *testing.T) {
dirs := []string{"h1/index-v0.11.0.db"}
// Create directories that reset will remove
for _, dir := range dirs {
err := os.Mkdir(dir, 0755)
if err != nil && !os.IsExist(err) {
t.Fatal(err)
}
}
// Run reset to clean up
cmd := exec.Command("../bin/syncthing", "-no-browser", "-home", "h1", "-reset")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stdout
err := cmd.Run()
if err != nil {
t.Fatal(err)
}
// Verify that they're gone
for _, dir := range dirs {
_, err := os.Stat(dir)
if err == nil {
t.Errorf("%s still exists", dir)
}
}
// Clean up
dirs, err = osutil.Glob("*.syncthing-reset-*")
if err != nil {
t.Fatal(err)
}
removeAll(dirs...)
}
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v Staggered) Archive(filePath string) error {
if debug {
l.Debugln("Waiting for lock on ", v.versionsPath)
}
v.mutex.Lock()
defer v.mutex.Unlock()
_, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
if debug {
l.Debugln("not archiving nonexistent file", filePath)
}
return nil
} else if err != nil {
return err
}
if _, err := os.Stat(v.versionsPath); err != nil {
if os.IsNotExist(err) {
if debug {
l.Debugln("creating versions dir", v.versionsPath)
}
osutil.MkdirAll(v.versionsPath, 0755)
osutil.HideFile(v.versionsPath)
} else {
return err
}
}
if debug {
l.Debugln("archiving", filePath)
}
file := filepath.Base(filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
if err != nil {
return err
}
dir := filepath.Join(v.versionsPath, inFolderPath)
err = osutil.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
return err
}
ver := taggedFilename(file, time.Now().Format(TimeFormat))
dst := filepath.Join(dir, ver)
if debug {
l.Debugln("moving to", dst)
}
err = osutil.Rename(filePath, dst)
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
newVersions, err := osutil.Glob(filepath.Join(dir, taggedFilename(file, TimeGlob)))
if err != nil {
l.Warnln("globbing:", err)
return nil
}
// Also according to the old file.ext~timestamp pattern.
oldVersions, err := osutil.Glob(filepath.Join(dir, file+"~"+TimeGlob))
if err != nil {
l.Warnln("globbing:", err)
return nil
}
// Use all the found filenames.
versions := append(oldVersions, newVersions...)
v.expire(uniqueSortedStrings(versions))
return nil
}
func TestConflictsIndexReset(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
err = os.Mkdir("s1", 0755)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir("s2", 0755)
if err != nil {
t.Fatal(err)
}
// Three files on s1
err = ioutil.WriteFile("s1/file1", []byte("hello\n"), 0644)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile("s1/file2", []byte("hello\n"), 0644)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile("s2/file3", []byte("hello\n"), 0644)
if err != nil {
t.Fatal(err)
}
// Let them sync
sender := startInstance(t, 1)
defer checkedStop(t, sender)
receiver := startInstance(t, 2)
defer checkedStop(t, receiver)
log.Println("Syncing...")
rc.AwaitSync("default", sender, receiver)
log.Println("Verifying...")
// s1 should have three files
files, err := osutil.Glob("s1/file*")
if err != nil {
t.Fatal(err)
}
if len(files) != 3 {
t.Errorf("Expected 3 files in s1 instead of %d", len(files))
}
// s2 should have three
files, err = osutil.Glob("s2/file*")
if err != nil {
t.Fatal(err)
}
if len(files) != 3 {
t.Errorf("Expected 3 files in s2 instead of %d", len(files))
}
log.Println("Updating...")
// change s2/file2 a few times, so that it's version counter increases.
// This will make the file on the cluster look newer than what we have
// locally after we rest the index, unless we have a fix for that.
for i := 0; i < 5; i++ {
err = ioutil.WriteFile("s2/file2", []byte("hello1\n"), 0644)
if err != nil {
t.Fatal(err)
}
err = receiver.Rescan("default")
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
}
rc.AwaitSync("default", sender, receiver)
// Now nuke the index
log.Println("Resetting...")
checkedStop(t, receiver)
removeAll("h2/index*")
// s1/file1 (remote) changes while receiver is down
err = ioutil.WriteFile("s1/file1", []byte("goodbye\n"), 0644)
if err != nil {
t.Fatal(err)
}
// s1 must know about it
err = sender.Rescan("default")
if err != nil {
t.Fatal(err)
}
// s2/file2 (local) changes while receiver is down
err = ioutil.WriteFile("s2/file2", []byte("goodbye\n"), 0644)
if err != nil {
t.Fatal(err)
}
receiver = startInstance(t, 2)
defer checkedStop(t, receiver)
log.Println("Syncing...")
rc.AwaitSync("default", sender, receiver)
// s2 should have five files (three plus two conflicts)
files, err = osutil.Glob("s2/file*")
if err != nil {
t.Fatal(err)
}
if len(files) != 5 {
t.Errorf("Expected 5 files in s2 instead of %d", len(files))
}
// file1 is in conflict, so there's two versions of that one
files, err = osutil.Glob("s2/file1*")
if err != nil {
t.Fatal(err)
}
if len(files) != 2 {
t.Errorf("Expected 2 'file1' files in s2 instead of %d", len(files))
}
// file2 is in conflict, so there's two versions of that one
files, err = osutil.Glob("s2/file2*")
if err != nil {
t.Fatal(err)
}
if len(files) != 2 {
t.Errorf("Expected 2 'file2' files in s2 instead of %d", len(files))
}
}
func TestConflictsDefault(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
log.Println("Generating files...")
err = generateFiles("s1", 100, 20, "../LICENSE")
if err != nil {
t.Fatal(err)
}
fd, err := os.Create("s1/testfile.txt")
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("hello\n")
if err != nil {
t.Fatal(err)
}
err = fd.Close()
if err != nil {
t.Fatal(err)
}
expected, err := directoryContents("s1")
if err != nil {
t.Fatal(err)
}
sender := startInstance(t, 1)
defer checkedStop(t, sender)
receiver := startInstance(t, 2)
defer checkedStop(t, receiver)
// Rescan with a delay on the next one, so we are not surprised by a
// sudden rescan while we're trying to introduce conflicts.
if err := sender.RescanDelay("default", 86400); err != nil {
t.Fatal(err)
}
if err := receiver.RescanDelay("default", 86400); err != nil {
t.Fatal(err)
}
rc.AwaitSync("default", sender, receiver)
log.Println("Verifying...")
actual, err := directoryContents("s2")
if err != nil {
t.Fatal(err)
}
err = compareDirectoryContents(actual, expected)
if err != nil {
t.Fatal(err)
}
log.Println("Introducing a conflict (simultaneous edit)...")
if err := sender.PauseDevice(receiver.ID()); err != nil {
t.Fatal(err)
}
fd, err = os.OpenFile("s1/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("text added to s1\n")
if err != nil {
t.Fatal(err)
}
err = fd.Close()
if err != nil {
t.Fatal(err)
}
fd, err = os.OpenFile("s2/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("text added to s2\n")
if err != nil {
t.Fatal(err)
}
err = fd.Close()
if err != nil {
t.Fatal(err)
}
if err := sender.ResumeDevice(receiver.ID()); err != nil {
t.Fatal(err)
}
log.Println("Syncing...")
if err := sender.RescanDelay("default", 86400); err != nil {
t.Fatal(err)
}
if err := receiver.RescanDelay("default", 86400); err != nil {
t.Fatal(err)
}
rc.AwaitSync("default", sender, receiver)
// The conflict is expected on the s2 side due to how we calculate which
// file is the winner (based on device ID)
files, err := osutil.Glob("s2/*sync-conflict*")
if err != nil {
t.Fatal(err)
}
if len(files) != 1 {
t.Errorf("Expected 1 conflicted files instead of %d", len(files))
}
log.Println("Introducing a conflict (edit plus delete)...")
if err := sender.PauseDevice(receiver.ID()); err != nil {
t.Fatal(err)
}
err = os.Remove("s1/testfile.txt")
if err != nil {
t.Fatal(err)
}
fd, err = os.OpenFile("s2/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("more text added to s2\n")
if err != nil {
t.Fatal(err)
}
err = fd.Close()
if err != nil {
t.Fatal(err)
}
if err := sender.ResumeDevice(receiver.ID()); err != nil {
t.Fatal(err)
}
log.Println("Syncing...")
if err := sender.RescanDelay("default", 86400); err != nil {
t.Fatal(err)
}
if err := receiver.RescanDelay("default", 86400); err != nil {
t.Fatal(err)
}
rc.AwaitSync("default", sender, receiver)
// The conflict is resolved to the advantage of the edit over the delete.
// As such, we get the edited content synced back to s1 where it was
// removed.
files, err = osutil.Glob("s2/*sync-conflict*")
if err != nil {
t.Fatal(err)
}
if len(files) != 1 {
t.Errorf("Expected 1 conflicted files instead of %d", len(files))
}
bs, err := ioutil.ReadFile("s1/testfile.txt")
if err != nil {
t.Error("reading file:", err)
}
if !bytes.Contains(bs, []byte("more text added to s2")) {
t.Error("s1/testfile.txt should contain data added in s2")
}
}
func TestConflictsInitialMerge(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
err = os.Mkdir("s1", 0755)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir("s2", 0755)
if err != nil {
t.Fatal(err)
}
// File 1 is a conflict
err = ioutil.WriteFile("s1/file1", []byte("hello\n"), 0644)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile("s2/file1", []byte("goodbye\n"), 0644)
if err != nil {
t.Fatal(err)
}
// File 2 exists on s1 only
err = ioutil.WriteFile("s1/file2", []byte("hello\n"), 0644)
if err != nil {
t.Fatal(err)
}
// File 3 exists on s2 only
err = ioutil.WriteFile("s2/file3", []byte("goodbye\n"), 0644)
if err != nil {
t.Fatal(err)
}
// Let them sync
sender := startInstance(t, 1)
defer checkedStop(t, sender)
receiver := startInstance(t, 2)
defer checkedStop(t, receiver)
log.Println("Syncing...")
rc.AwaitSync("default", sender, receiver)
checkedStop(t, sender)
checkedStop(t, receiver)
log.Println("Verifying...")
// s1 should have three-four files (there's a conflict from s2 which may or may not have synced yet)
files, err := osutil.Glob("s1/file*")
if err != nil {
t.Fatal(err)
}
if len(files) < 3 || len(files) > 4 {
t.Errorf("Expected 3-4 files in s1 instead of %d", len(files))
}
// s2 should have four files (there's a conflict)
files, err = osutil.Glob("s2/file*")
if err != nil {
t.Fatal(err)
}
if len(files) != 4 {
t.Errorf("Expected 4 files in s2 instead of %d", len(files))
}
// file1 is in conflict, so there's two versions of that one
files, err = osutil.Glob("s2/file1*")
if err != nil {
t.Fatal(err)
}
if len(files) != 2 {
t.Errorf("Expected 2 'file1' files in s2 instead of %d", len(files))
}
}
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v Simple) Archive(filePath string) error {
fileInfo, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
versionsDir := filepath.Join(v.folderPath, ".stversions")
_, err = os.Stat(versionsDir)
if err != nil {
if os.IsNotExist(err) {
l.Debugln("creating versions dir", versionsDir)
osutil.MkdirAll(versionsDir, 0755)
osutil.HideFile(versionsDir)
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
if err != nil {
return err
}
dir := filepath.Join(versionsDir, inFolderPath)
err = osutil.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
return err
}
ver := taggedFilename(file, fileInfo.ModTime().Format(TimeFormat))
dst := filepath.Join(dir, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(filePath, dst)
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
newVersions, err := osutil.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
}
// Also according to the old file.ext~timestamp pattern.
pattern = filepath.Join(dir, file+"~"+TimeGlob)
oldVersions, err := osutil.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
}
// Use all the found filenames. "~" sorts after "." so all old pattern
// files will be deleted before any new, which is as it should be.
versions := uniqueSortedStrings(append(oldVersions, newVersions...))
if len(versions) > v.keep {
for _, toRemove := range versions[:len(versions)-v.keep] {
l.Debugln("cleaning out", toRemove)
err = os.Remove(toRemove)
if err != nil {
l.Warnln("removing old version:", err)
}
}
}
return nil
}