func TestIgnores(t *testing.T) {
// Clean and start a syncthing instance
log.Println("Cleaning...")
err := removeAll("s1", "h1/index")
if err != nil {
t.Fatal(err)
}
p := syncthingProcess{ // id1
instance: "1",
argv: []string{"-home", "h1"},
port: 8081,
apiKey: apiKey,
}
err = p.start()
if err != nil {
t.Fatal(err)
}
defer p.stop()
// Create eight empty files and directories
files := []string{"f1", "f2", "f3", "f4", "f11", "f12", "f13", "f14"}
dirs := []string{"d1", "d2", "d3", "d4", "d11", "d12", "d13", "d14"}
all := append(files, dirs...)
for _, file := range files {
fd, err := os.Create(filepath.Join("s1", file))
if err != nil {
t.Fatal(err)
}
fd.Close()
}
for _, dir := range dirs {
err := os.Mkdir(filepath.Join("s1", dir), 0755)
if err != nil {
t.Fatal(err)
}
}
var syms []string
if symlinksSupported() {
syms = []string{"s1", "s2", "s3", "s4", "s11", "s12", "s13", "s14"}
for _, sym := range syms {
p := filepath.Join("s1", sym)
symlinks.Create(p, p, 0)
}
all = append(all, syms...)
}
// Rescan and verify that we see them all
p.post("/rest/scan?folder=default", nil)
m, err := p.model("default")
if err != nil {
t.Fatal(err)
}
expected := len(all) // nothing is ignored
if m.LocalFiles != expected {
t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
}
// Add some of them to an ignore file
fd, err := os.Create("s1/.stignore")
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("f1*\nf2\nd1*\nd2\ns1*\ns2") // [fds][34] only non-ignored items
if err != nil {
t.Fatal(err)
}
err = fd.Close()
if err != nil {
t.Fatal(err)
}
// Rescan and verify that we see them
p.post("/rest/scan?folder=default", nil)
m, err = p.model("default")
if err != nil {
t.Fatal(err)
}
expected = len(all) * 2 / 8 // two out of eight items of each type should remain
if m.LocalFiles != expected {
t.Fatalf("Incorrect number of files after first ignore, %d != %d", m.LocalFiles, expected)
}
// Change the pattern to include some of the files and dirs previously ignored
fd, err = os.Create("s1/.stignore")
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("f2\nd2\ns2\n")
if err != nil {
t.Fatal(err)
}
err = fd.Close()
if err != nil {
t.Fatal(err)
}
// Rescan and verify that we see them
p.post("/rest/scan?folder=default", nil)
m, err = p.model("default")
if err != nil {
t.Fatal(err)
}
expected = len(all) * 7 / 8 // seven out of eight items of each type should remain
if m.LocalFiles != expected {
t.Fatalf("Incorrect number of files after second ignore, %d != %d", m.LocalFiles, expected)
}
}
func TestSymlinks(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
if err != nil {
t.Fatal(err)
}
log.Println("Generating files...")
err = generateFiles("s1", 100, 20, "../LICENSE")
if err != nil {
t.Fatal(err)
}
// A file that we will replace with a symlink later
fd, err := os.Create("s1/fileToReplace")
if err != nil {
t.Fatal(err)
}
fd.Close()
// A directory that we will replace with a symlink later
err = os.Mkdir("s1/dirToReplace", 0755)
if err != nil {
t.Fatal(err)
}
// A file and a symlink to that file
fd, err = os.Create("s1/file")
if err != nil {
t.Fatal(err)
}
fd.Close()
err = symlinks.Create("s1/fileLink", "file", 0)
if err != nil {
log.Fatal(err)
}
// A directory and a symlink to that directory
err = os.Mkdir("s1/dir", 0755)
if err != nil {
t.Fatal(err)
}
err = symlinks.Create("s1/dirLink", "dir", 0)
if err != nil {
log.Fatal(err)
}
// A link to something in the repo that does not exist
err = symlinks.Create("s1/noneLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will replace with a file later
err = symlinks.Create("s1/repFileLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will replace with a directory later
err = symlinks.Create("s1/repDirLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// Verify that the files and symlinks sync to the other side
log.Println("Syncing...")
sender := syncthingProcess{ // id1
log: "1.out",
argv: []string{"-home", "h1"},
port: 8081,
apiKey: apiKey,
}
err = sender.start()
if err != nil {
t.Fatal(err)
}
receiver := syncthingProcess{ // id2
log: "2.out",
argv: []string{"-home", "h2"},
port: 8082,
apiKey: apiKey,
}
err = receiver.start()
if err != nil {
_ = sender.stop()
t.Fatal(err)
}
for {
comp, err := sender.peerCompletion()
if err != nil {
if strings.Contains(err.Error(), "use of closed network connection") {
time.Sleep(time.Second)
continue
}
_ = sender.stop()
_ = receiver.stop()
t.Fatal(err)
}
curComp := comp[id2]
if curComp == 100 {
break
}
time.Sleep(time.Second)
}
err = sender.stop()
if err != nil {
t.Fatal(err)
}
err = receiver.stop()
if err != nil {
t.Fatal(err)
}
log.Println("Comparing directories...")
err = compareDirectories("s1", "s2")
if err != nil {
t.Fatal(err)
}
log.Println("Making some changes...")
// Remove one symlink
err = os.Remove("s1/fileLink")
if err != nil {
log.Fatal(err)
}
// Change the target of another
err = os.Remove("s1/dirLink")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/dirLink", "file", 0)
if err != nil {
log.Fatal(err)
}
// Replace one with a file
err = os.Remove("s1/repFileLink")
if err != nil {
log.Fatal(err)
}
fd, err = os.Create("s1/repFileLink")
if err != nil {
log.Fatal(err)
}
fd.Close()
// Replace one with a directory
err = os.Remove("s1/repDirLink")
if err != nil {
log.Fatal(err)
}
err = os.Mkdir("s1/repDirLink", 0755)
if err != nil {
log.Fatal(err)
}
// Replace a file with a symlink
err = os.Remove("s1/fileToReplace")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/fileToReplace", "somewhere/non/existent", 0)
if err != nil {
log.Fatal(err)
}
// Replace a directory with a symlink
err = os.RemoveAll("s1/dirToReplace")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/dirToReplace", "somewhere/non/existent", 0)
if err != nil {
log.Fatal(err)
}
// Sync these changes and recheck
log.Println("Syncing...")
err = sender.start()
if err != nil {
t.Fatal(err)
}
err = receiver.start()
if err != nil {
_ = sender.stop()
t.Fatal(err)
}
for {
comp, err := sender.peerCompletion()
if err != nil {
if strings.Contains(err.Error(), "use of closed network connection") {
time.Sleep(time.Second)
continue
}
_ = sender.stop()
_ = receiver.stop()
t.Fatal(err)
}
curComp := comp[id2]
if curComp == 100 {
break
}
time.Sleep(time.Second)
}
err = sender.stop()
if err != nil {
t.Fatal(err)
}
err = receiver.stop()
if err != nil {
t.Fatal(err)
}
log.Println("Comparing directories...")
err = compareDirectories("s1", "s2")
if err != nil {
t.Fatal(err)
}
}
func (p *rwFolder) performFinish(state *sharedPullerState) error {
// Set the correct permission bits on the new file
if !p.ignorePermissions(state.file) {
if err := os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777)); err != nil {
return err
}
}
// Set the correct timestamp on the new file
t := time.Unix(state.file.Modified, 0)
if err := os.Chtimes(state.tempName, t, t); err != nil {
// Try using virtual mtimes instead
info, err := os.Stat(state.tempName)
if err != nil {
return err
}
p.virtualMtimeRepo.UpdateMtime(state.file.Name, info.ModTime(), t)
}
var err error
if p.inConflict(state.version, state.file.Version) {
// The new file has been changed in conflict with the existing one. We
// should file it away as a conflict instead of just removing or
// archiving. Also merge with the version vector we had, to indicate
// we have resolved the conflict.
state.file.Version = state.file.Version.Merge(state.version)
err = osutil.InWritableDir(moveForConflict, state.realName)
} else if p.versioner != nil {
// If we should use versioning, let the versioner archive the old
// file before we replace it. Archiving a non-existent file is not
// an error.
err = p.versioner.Archive(state.realName)
} else {
err = nil
}
if err != nil {
return err
}
// If the target path is a symlink or a directory, we cannot copy
// over it, hence remove it before proceeding.
stat, err := osutil.Lstat(state.realName)
if err == nil && (stat.IsDir() || stat.Mode()&os.ModeSymlink != 0) {
osutil.InWritableDir(osutil.Remove, state.realName)
}
// Replace the original content with the new one
if err = osutil.Rename(state.tempName, state.realName); err != nil {
return err
}
// If it's a symlink, the target of the symlink is inside the file.
if state.file.IsSymlink() {
content, err := ioutil.ReadFile(state.realName)
if err != nil {
return err
}
// Remove the file, and replace it with a symlink.
err = osutil.InWritableDir(func(path string) error {
os.Remove(path)
return symlinks.Create(path, string(content), state.file.Flags)
}, state.realName)
if err != nil {
return err
}
}
// Record the updated file in the index
p.dbUpdates <- dbUpdateJob{state.file, dbUpdateHandleFile}
return nil
}
func testSymlinks(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
log.Println("Generating files...")
err = generateFiles("s1", 100, 20, "../LICENSE")
if err != nil {
t.Fatal(err)
}
// A file that we will replace with a symlink later
fd, err := os.Create("s1/fileToReplace")
if err != nil {
t.Fatal(err)
}
fd.Close()
// A directory that we will replace with a symlink later
err = os.Mkdir("s1/dirToReplace", 0755)
if err != nil {
t.Fatal(err)
}
// A file and a symlink to that file
fd, err = os.Create("s1/file")
if err != nil {
t.Fatal(err)
}
fd.Close()
err = symlinks.Create("s1/fileLink", "file", 0)
if err != nil {
log.Fatal(err)
}
// A directory and a symlink to that directory
err = os.Mkdir("s1/dir", 0755)
if err != nil {
t.Fatal(err)
}
err = symlinks.Create("s1/dirLink", "dir", 0)
if err != nil {
log.Fatal(err)
}
// A link to something in the repo that does not exist
err = symlinks.Create("s1/noneLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will replace with a file later
err = symlinks.Create("s1/repFileLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will replace with a directory later
err = symlinks.Create("s1/repDirLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will remove later
err = symlinks.Create("s1/removeLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// Verify that the files and symlinks sync to the other side
sender := startInstance(t, 1)
defer checkedStop(t, sender)
receiver := startInstance(t, 2)
defer checkedStop(t, receiver)
log.Println("Syncing...")
rc.AwaitSync("default", sender, receiver)
log.Println("Comparing directories...")
err = compareDirectories("s1", "s2")
if err != nil {
t.Fatal(err)
}
log.Println("Making some changes...")
// Remove one symlink
err = os.Remove("s1/fileLink")
if err != nil {
log.Fatal(err)
}
// Change the target of another
err = os.Remove("s1/dirLink")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/dirLink", "file", 0)
if err != nil {
log.Fatal(err)
}
// Replace one with a file
err = os.Remove("s1/repFileLink")
if err != nil {
log.Fatal(err)
}
fd, err = os.Create("s1/repFileLink")
if err != nil {
log.Fatal(err)
}
fd.Close()
// Replace one with a directory
err = os.Remove("s1/repDirLink")
if err != nil {
log.Fatal(err)
}
err = os.Mkdir("s1/repDirLink", 0755)
if err != nil {
log.Fatal(err)
}
// Replace a file with a symlink
err = os.Remove("s1/fileToReplace")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/fileToReplace", "somewhere/non/existent", 0)
if err != nil {
log.Fatal(err)
}
// Replace a directory with a symlink
err = os.RemoveAll("s1/dirToReplace")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/dirToReplace", "somewhere/non/existent", 0)
if err != nil {
log.Fatal(err)
}
// Remove a broken symlink
err = os.Remove("s1/removeLink")
if err != nil {
log.Fatal(err)
}
// Sync these changes and recheck
log.Println("Syncing...")
if err := sender.Rescan("default"); err != nil {
t.Fatal(err)
}
rc.AwaitSync("default", sender, receiver)
log.Println("Comparing directories...")
err = compareDirectories("s1", "s2")
if err != nil {
t.Fatal(err)
}
}
func (p *Puller) performFinish(state *sharedPullerState) {
// Verify the file against expected hashes
fd, err := os.Open(state.tempName)
if err != nil {
l.Warnln("puller: final:", err)
return
}
err = scanner.Verify(fd, protocol.BlockSize, state.file.Blocks)
fd.Close()
if err != nil {
l.Infoln("puller:", state.file.Name, err, "(file changed during pull?)")
return
}
// Set the correct permission bits on the new file
if !p.ignorePerms {
err = os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777))
if err != nil {
l.Warnln("puller: final:", err)
return
}
}
// Set the correct timestamp on the new file
t := time.Unix(state.file.Modified, 0)
err = os.Chtimes(state.tempName, t, t)
if err != nil {
if p.lenientMtimes {
// We accept the failure with a warning here and allow the sync to
// continue. We'll sync the new mtime back to the other devices later.
// If they have the same problem & setting, we might never get in
// sync.
l.Infof("Puller (folder %q, file %q): final: %v (continuing anyway as requested)", p.folder, state.file.Name, err)
} else {
l.Warnln("puller: final:", err)
return
}
}
// If we should use versioning, let the versioner archive the old
// file before we replace it. Archiving a non-existent file is not
// an error.
if p.versioner != nil {
err = p.versioner.Archive(state.realName)
if err != nil {
l.Warnln("puller: final:", err)
return
}
}
// If the target path is a symlink or a directory, we cannot copy
// over it, hence remove it before proceeding.
stat, err := os.Lstat(state.realName)
isLink, _ := symlinks.IsSymlink(state.realName)
if isLink || (err == nil && stat.IsDir()) {
osutil.InWritableDir(os.Remove, state.realName)
}
// Replace the original content with the new one
err = osutil.Rename(state.tempName, state.realName)
if err != nil {
l.Warnln("puller: final:", err)
return
}
// If it's a symlink, the target of the symlink is inside the file.
if state.file.IsSymlink() {
content, err := ioutil.ReadFile(state.realName)
if err != nil {
l.Warnln("puller: final: reading symlink:", err)
return
}
// Remove the file, and replace it with a symlink.
err = osutil.InWritableDir(func(path string) error {
os.Remove(path)
return symlinks.Create(path, string(content), state.file.Flags)
}, state.realName)
if err != nil {
l.Warnln("puller: final: creating symlink:", err)
return
}
}
// Record the updated file in the index
p.model.updateLocal(p.folder, state.file)
}