func TestManyPeers(t *testing.T) { log.Println("Cleaning...") err := removeAll("s1", "s2", "h1/index*", "h2/index*") if err != nil { t.Fatal(err) } log.Println("Generating files...") err = generateFiles("s1", 200, 20, "../LICENSE") if err != nil { t.Fatal(err) } receiver := startInstance(t, 2) defer checkedStop(t, receiver) bs, err := receiver.Get("/rest/system/config") if err != nil { t.Fatal(err) } var cfg config.Configuration if err := json.Unmarshal(bs, &cfg); err != nil { t.Fatal(err) } for len(cfg.Devices) < 100 { bs := make([]byte, 16) ReadRand(bs) id := protocol.NewDeviceID(bs) cfg.Devices = append(cfg.Devices, config.DeviceConfiguration{DeviceID: id}) cfg.Folders[0].Devices = append(cfg.Folders[0].Devices, config.FolderDeviceConfiguration{DeviceID: id}) } osutil.Rename("h2/config.xml", "h2/config.xml.orig") defer osutil.Rename("h2/config.xml.orig", "h2/config.xml") var buf bytes.Buffer json.NewEncoder(&buf).Encode(cfg) _, err = receiver.Post("/rest/system/config", &buf) if err != nil { t.Fatal(err) } sender := startInstance(t, 1) defer checkedStop(t, sender) rc.AwaitSync("default", sender, receiver) log.Println("Comparing directories...") err = compareDirectories("s1", "s2") if err != nil { t.Fatal(err) } }
func archiveAndSaveConfig(cfg *config.Wrapper) error { // To prevent previous config from being cleaned up, quickly touch it too now := time.Now() _ = os.Chtimes(cfg.ConfigPath(), now, now) // May return error on Android etc; no worries archivePath := cfg.ConfigPath() + fmt.Sprintf(".v%d", cfg.Raw().OriginalVersion) l.Infoln("Archiving a copy of old config file format at:", archivePath) if err := osutil.Rename(cfg.ConfigPath(), archivePath); err != nil { return err } return cfg.Save() }
func TestInWritableDirWindowsRename(t *testing.T) { if runtime.GOOS != "windows" { t.Skipf("Tests not required") return } err := os.RemoveAll("testdata") if err != nil { t.Fatal(err) } defer os.RemoveAll("testdata") create := func(name string) error { fd, err := os.Create(name) if err != nil { return err } fd.Close() return nil } os.Mkdir("testdata", 0700) os.Mkdir("testdata/windows", 0500) os.Mkdir("testdata/windows/ro", 0500) create("testdata/windows/ro/readonly") os.Chmod("testdata/windows/ro/readonly", 0500) for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} { err := os.Rename(path, path+"new") if err == nil { t.Skipf("seem like this test doesn't work here") return } } rename := func(path string) error { return osutil.Rename(path, path+"new") } for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} { err := osutil.InWritableDir(rename, path) if err != nil { t.Errorf("Unexpected error %s: %s", path, err) } _, err = os.Stat(path + "new") if err != nil { t.Errorf("Unexpected error %s: %s", path, err) } } }
// Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (t *Trashcan) Archive(filePath string) error { _, err := osutil.Lstat(filePath) if os.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } versionsDir := filepath.Join(t.folderPath, ".stversions") if _, err := os.Stat(versionsDir); err != nil { if !os.IsNotExist(err) { return err } l.Debugln("creating versions dir", versionsDir) if err := osutil.MkdirAll(versionsDir, 0777); err != nil { return err } osutil.HideFile(versionsDir) } l.Debugln("archiving", filePath) relativePath, err := filepath.Rel(t.folderPath, filePath) if err != nil { return err } archivedPath := filepath.Join(versionsDir, relativePath) if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) { return err } l.Debugln("moving to", archivedPath) if err := osutil.Rename(filePath, archivedPath); err != nil { return err } // Set the mtime to the time the file was deleted. This is used by the // cleanout routine. If this fails things won't work optimally but there's // not much we can do about it so we ignore the error. os.Chtimes(archivedPath, time.Now(), time.Now()) return nil }
// checkConvertDatabase tries to convert an existing old (v0.11) database to // new (v0.13) format. func checkConvertDatabase(dbFile string) error { oldLoc := filepath.Join(filepath.Dir(dbFile), "index-v0.11.0.db") if _, err := os.Stat(oldLoc); os.IsNotExist(err) { // The old database file does not exist; that's ok, continue as if // everything succeeded. return nil } else if err != nil { // Any other error is weird. return err } // There exists a database in the old format. We run a one time // conversion from old to new. fromDb, err := leveldb.OpenFile(oldLoc, nil) if err != nil { return err } toDb, err := leveldb.OpenFile(dbFile, nil) if err != nil { return err } err = convertKeyFormat(fromDb, toDb) if err != nil { return err } err = toDb.Close() if err != nil { return err } // We've done this one, we don't want to do it again (if the user runs // -reset or so). We don't care too much about errors any more at this stage. fromDb.Close() osutil.Rename(oldLoc, oldLoc+".converted") return nil }
func (p *rwFolder) performFinish(state *sharedPullerState) error { // Set the correct permission bits on the new file if !p.ignorePermissions(state.file) { if err := os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777)); err != nil { return err } } // Set the correct timestamp on the new file t := time.Unix(state.file.Modified, 0) if err := os.Chtimes(state.tempName, t, t); err != nil { // Try using virtual mtimes instead info, err := os.Stat(state.tempName) if err != nil { return err } p.virtualMtimeRepo.UpdateMtime(state.file.Name, info.ModTime(), t) } if stat, err := osutil.Lstat(state.realName); err == nil { // There is an old file or directory already in place. We need to // handle that. switch { case stat.IsDir() || stat.Mode()&os.ModeSymlink != 0: // It's a directory or a symlink. These are not versioned or // archived for conflicts, only removed (which of course fails for // non-empty directories). // TODO: This is the place where we want to remove temporary files // and future hard ignores before attempting a directory delete. // Should share code with p.deletDir(). if err = osutil.InWritableDir(osutil.Remove, state.realName); err != nil { return err } case p.inConflict(state.version, state.file.Version): // The new file has been changed in conflict with the existing one. We // should file it away as a conflict instead of just removing or // archiving. Also merge with the version vector we had, to indicate // we have resolved the conflict. state.file.Version = state.file.Version.Merge(state.version) if err = osutil.InWritableDir(p.moveForConflict, state.realName); err != nil { return err } case p.versioner != nil: // If we should use versioning, let the versioner archive the old // file before we replace it. Archiving a non-existent file is not // an error. if err = p.versioner.Archive(state.realName); err != nil { return err } } } // Replace the original content with the new one if err := osutil.Rename(state.tempName, state.realName); err != nil { return err } // If it's a symlink, the target of the symlink is inside the file. if state.file.IsSymlink() { content, err := ioutil.ReadFile(state.realName) if err != nil { return err } // Remove the file, and replace it with a symlink. err = osutil.InWritableDir(func(path string) error { os.Remove(path) tt := symlinks.TargetFile if state.file.IsDirectory() { tt = symlinks.TargetDirectory } return symlinks.Create(path, string(content), tt) }, state.realName) if err != nil { return err } } // Record the updated file in the index p.dbUpdates <- dbUpdateJob{state.file, dbUpdateHandleFile} return nil }
// Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (v Simple) Archive(filePath string) error { fileInfo, err := osutil.Lstat(filePath) if os.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } versionsDir := filepath.Join(v.folderPath, ".stversions") _, err = os.Stat(versionsDir) if err != nil { if os.IsNotExist(err) { l.Debugln("creating versions dir", versionsDir) osutil.MkdirAll(versionsDir, 0755) osutil.HideFile(versionsDir) } else { return err } } l.Debugln("archiving", filePath) file := filepath.Base(filePath) inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath)) if err != nil { return err } dir := filepath.Join(versionsDir, inFolderPath) err = osutil.MkdirAll(dir, 0755) if err != nil && !os.IsExist(err) { return err } ver := taggedFilename(file, fileInfo.ModTime().Format(TimeFormat)) dst := filepath.Join(dir, ver) l.Debugln("moving to", dst) err = osutil.Rename(filePath, dst) if err != nil { return err } // Glob according to the new file~timestamp.ext pattern. pattern := filepath.Join(dir, taggedFilename(file, TimeGlob)) newVersions, err := osutil.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil } // Also according to the old file.ext~timestamp pattern. pattern = filepath.Join(dir, file+"~"+TimeGlob) oldVersions, err := osutil.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil } // Use all the found filenames. "~" sorts after "." so all old pattern // files will be deleted before any new, which is as it should be. versions := uniqueSortedStrings(append(oldVersions, newVersions...)) if len(versions) > v.keep { for _, toRemove := range versions[:len(versions)-v.keep] { l.Debugln("cleaning out", toRemove) err = os.Remove(toRemove) if err != nil { l.Warnln("removing old version:", err) } } } return nil }
func TestOverride(t *testing.T) { // Enable "Master" on s1/default id, _ := protocol.DeviceIDFromString(id1) cfg, _ := config.Load("h1/config.xml", id) fld := cfg.Folders()["default"] fld.ReadOnly = true cfg.SetFolder(fld) os.Rename("h1/config.xml", "h1/config.xml.orig") defer osutil.Rename("h1/config.xml.orig", "h1/config.xml") cfg.Save() log.Println("Cleaning...") err := removeAll("s1", "s2", "h1/index*", "h2/index*") if err != nil { t.Fatal(err) } log.Println("Generating files...") err = generateFiles("s1", 100, 20, "../LICENSE") if err != nil { t.Fatal(err) } fd, err := os.Create("s1/testfile.txt") if err != nil { t.Fatal(err) } _, err = fd.WriteString("hello\n") if err != nil { t.Fatal(err) } err = fd.Close() if err != nil { t.Fatal(err) } expected, err := directoryContents("s1") if err != nil { t.Fatal(err) } master := startInstance(t, 1) defer checkedStop(t, master) slave := startInstance(t, 2) defer checkedStop(t, slave) log.Println("Syncing...") rc.AwaitSync("default", master, slave) log.Println("Verifying...") actual, err := directoryContents("s2") if err != nil { t.Fatal(err) } err = compareDirectoryContents(actual, expected) if err != nil { t.Fatal(err) } log.Println("Changing file on slave side...") fd, err = os.OpenFile("s2/testfile.txt", os.O_WRONLY|os.O_APPEND, 0644) if err != nil { t.Fatal(err) } _, err = fd.WriteString("text added to s2\n") if err != nil { t.Fatal(err) } err = fd.Close() if err != nil { t.Fatal(err) } if err := slave.Rescan("default"); err != nil { t.Fatal(err) } log.Println("Waiting for index to send...") time.Sleep(10 * time.Second) log.Println("Hitting Override on master...") if _, err := master.Post("/rest/db/override?folder=default", nil); err != nil { t.Fatal(err) } log.Println("Syncing...") rc.AwaitSync("default", master, slave) // Verify that the override worked fd, err = os.Open("s1/testfile.txt") if err != nil { t.Fatal(err) } bs, err := ioutil.ReadAll(fd) if err != nil { t.Fatal(err) } fd.Close() if strings.Contains(string(bs), "added to s2") { t.Error("Change should not have been synced to master") } fd, err = os.Open("s2/testfile.txt") if err != nil { t.Fatal(err) } bs, err = ioutil.ReadAll(fd) if err != nil { t.Fatal(err) } fd.Close() if strings.Contains(string(bs), "added to s2") { t.Error("Change should have been overridden on slave") } }
// Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (v Staggered) Archive(filePath string) error { l.Debugln("Waiting for lock on ", v.versionsPath) v.mutex.Lock() defer v.mutex.Unlock() _, err := osutil.Lstat(filePath) if os.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } if _, err := os.Stat(v.versionsPath); err != nil { if os.IsNotExist(err) { l.Debugln("creating versions dir", v.versionsPath) osutil.MkdirAll(v.versionsPath, 0755) osutil.HideFile(v.versionsPath) } else { return err } } l.Debugln("archiving", filePath) file := filepath.Base(filePath) inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath)) if err != nil { return err } dir := filepath.Join(v.versionsPath, inFolderPath) err = osutil.MkdirAll(dir, 0755) if err != nil && !os.IsExist(err) { return err } ver := taggedFilename(file, time.Now().Format(TimeFormat)) dst := filepath.Join(dir, ver) l.Debugln("moving to", dst) err = osutil.Rename(filePath, dst) if err != nil { return err } // Glob according to the new file~timestamp.ext pattern. pattern := filepath.Join(dir, taggedFilename(file, TimeGlob)) newVersions, err := osutil.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil } // Also according to the old file.ext~timestamp pattern. pattern = filepath.Join(dir, file+"~"+TimeGlob) oldVersions, err := osutil.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil } // Use all the found filenames. versions := append(oldVersions, newVersions...) v.expire(uniqueSortedStrings(versions)) return nil }
func TestFolderWithoutRestart(t *testing.T) { log.Println("Cleaning...") err := removeAll("testfolder-p1", "testfolder-p4", "h1/index*", "h4/index*") if err != nil { t.Fatal(err) } defer removeAll("testfolder-p1", "testfolder-p4") if err := generateFiles("testfolder-p1", 50, 18, "../LICENSE"); err != nil { t.Fatal(err) } p1 := startInstance(t, 1) defer checkedStop(t, p1) p4 := startInstance(t, 4) defer checkedStop(t, p4) if ok, err := p1.ConfigInSync(); err != nil || !ok { t.Fatal("p1 should be in sync;", ok, err) } if ok, err := p4.ConfigInSync(); err != nil || !ok { t.Fatal("p4 should be in sync;", ok, err) } // Add a new folder to p1, shared with p4. Back up and restore the config // first. log.Println("Adding testfolder to p1...") os.Remove("h1/config.xml.orig") os.Rename("h1/config.xml", "h1/config.xml.orig") defer osutil.Rename("h1/config.xml.orig", "h1/config.xml") cfg, err := p1.GetConfig() if err != nil { t.Fatal(err) } newFolder := config.FolderConfiguration{ ID: "testfolder", RawPath: "testfolder-p1", RescanIntervalS: 86400, Copiers: 1, Hashers: 1, Pullers: 1, Devices: []config.FolderDeviceConfiguration{{DeviceID: p4.ID()}}, } newDevice := config.DeviceConfiguration{ DeviceID: p4.ID(), Name: "p4", Addresses: []string{"dynamic"}, Compression: protocol.CompressMetadata, } cfg.Folders = append(cfg.Folders, newFolder) cfg.Devices = append(cfg.Devices, newDevice) if err = p1.PostConfig(cfg); err != nil { t.Fatal(err) } // Add a new folder to p4, shared with p1. Back up and restore the config // first. log.Println("Adding testfolder to p4...") os.Remove("h4/config.xml.orig") os.Rename("h4/config.xml", "h4/config.xml.orig") defer osutil.Rename("h4/config.xml.orig", "h4/config.xml") cfg, err = p4.GetConfig() if err != nil { t.Fatal(err) } newFolder.RawPath = "testfolder-p4" newFolder.Devices = []config.FolderDeviceConfiguration{{DeviceID: p1.ID()}} newDevice.DeviceID = p1.ID() newDevice.Name = "p1" newDevice.Addresses = []string{"127.0.0.1:22001"} cfg.Folders = append(cfg.Folders, newFolder) cfg.Devices = append(cfg.Devices, newDevice) if err = p4.PostConfig(cfg); err != nil { t.Fatal(err) } // The change should not require a restart, so the config should be "in sync" if ok, err := p1.ConfigInSync(); err != nil || !ok { t.Fatal("p1 should be in sync;", ok, err) } if ok, err := p4.ConfigInSync(); err != nil || !ok { t.Fatal("p4 should be in sync;", ok, err) } // The folder should start and scan - wait for the event that signals this // has happened. log.Println("Waiting for testfolder to scan...") since := 0 outer: for { events, err := p4.Events(since) if err != nil { t.Fatal(err) } for _, event := range events { if event.Type == "StateChanged" { data := event.Data.(map[string]interface{}) folder := data["folder"].(string) from := data["from"].(string) to := data["to"].(string) if folder == "testfolder" && from == "scanning" && to == "idle" { break outer } } since = event.ID } } // It should sync to the other side successfully log.Println("Waiting for p1 and p4 to connect and sync...") rc.AwaitSync("testfolder", p1, p4) }
func TestAddDeviceWithoutRestart(t *testing.T) { log.Println("Cleaning...") err := removeAll("s1", "h1/index*", "s4", "h4/index*") if err != nil { t.Fatal(err) } log.Println("Generating files...") err = generateFiles("s1", 100, 18, "../LICENSE") if err != nil { t.Fatal(err) } p1 := startInstance(t, 1) defer checkedStop(t, p1) p4 := startInstance(t, 4) defer checkedStop(t, p4) if ok, err := p1.ConfigInSync(); err != nil || !ok { t.Fatal("p1 should be in sync;", ok, err) } if ok, err := p4.ConfigInSync(); err != nil || !ok { t.Fatal("p4 should be in sync;", ok, err) } // Add the p1 device to p4. Back up and restore p4's config first. log.Println("Adding p1 to p4...") os.Remove("h4/config.xml.orig") os.Rename("h4/config.xml", "h4/config.xml.orig") defer osutil.Rename("h4/config.xml.orig", "h4/config.xml") cfg, err := p4.GetConfig() if err != nil { t.Fatal(err) } devCfg := config.DeviceConfiguration{ DeviceID: p1.ID(), Name: "s1", Addresses: []string{"127.0.0.1:22001"}, Compression: protocol.CompressMetadata, } cfg.Devices = append(cfg.Devices, devCfg) cfg.Folders[0].Devices = append(cfg.Folders[0].Devices, config.FolderDeviceConfiguration{DeviceID: p1.ID()}) if err = p4.PostConfig(cfg); err != nil { t.Fatal(err) } // The change should not require a restart, so the config should be "in sync" if ok, err := p4.ConfigInSync(); err != nil || !ok { t.Fatal("p4 should be in sync;", ok, err) } // Wait for the devices to connect and sync. log.Println("Waiting for p1 and p4 to connect and sync...") rc.AwaitSync("default", p1, p4) }