// Test with exclude and delete excluded
func TestSyncWithExcludeAndDeleleteExcluded(t *testing.T) {
fs.Config.Filter.MaxSize = 40
fs.Config.Filter.DeleteExcluded = true
reset := func() {
fs.Config.Filter.MaxSize = 0
fs.Config.Filter.DeleteExcluded = false
}
defer reset()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Tidy up
reset()
err = os.Remove(localName + "/enormous")
if err != nil {
t.Fatalf("Remove failed: %v", err)
}
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items = []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
func TestRmdirs(t *testing.T) {
r := NewRun(t)
defer r.Finalise()
r.Mkdir(r.fremote)
// Clean any directories that have crept in so far
// FIXME make the Finalise method do this?
require.NoError(t, fs.Rmdirs(r.fremote))
// Make some files and dirs we expect to keep
r.ForceMkdir(r.fremote)
file1 := r.WriteObject("A1/B1/C1/one", "aaa", t1)
file2 := r.WriteObject("A1/two", "bbb", t2)
//..and dirs we expect to delete
require.NoError(t, fs.Mkdir(r.fremote, "A2"))
require.NoError(t, fs.Mkdir(r.fremote, "A1/B2"))
require.NoError(t, fs.Mkdir(r.fremote, "A1/B2/C2"))
require.NoError(t, fs.Mkdir(r.fremote, "A1/B1/C3"))
require.NoError(t, fs.Mkdir(r.fremote, "A3"))
require.NoError(t, fs.Mkdir(r.fremote, "A3/B3"))
require.NoError(t, fs.Mkdir(r.fremote, "A3/B3/C4"))
fstest.CheckListingWithPrecision(
t,
r.fremote,
[]fstest.Item{
file1, file2,
},
[]string{
"A1",
"A1/B1",
"A1/B1/C1",
"A2",
"A1/B2",
"A1/B2/C2",
"A1/B1/C3",
"A3",
"A3/B3",
"A3/B3/C4",
},
fs.Config.ModifyWindow,
)
require.NoError(t, fs.Rmdirs(r.fremote))
fstest.CheckListingWithPrecision(
t,
r.fremote,
[]fstest.Item{
file1, file2,
},
[]string{
"A1",
"A1/B1",
"A1/B1/C1",
},
fs.Config.ModifyWindow,
)
}
// Test with exclude
func TestSyncWithExclude(t *testing.T) {
WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fs.Config.Filter.MaxSize = 40
defer func() {
fs.Config.Filter.MaxSize = 0
}()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Now sync the other way round and check enormous doesn't get
// deleted as it is excluded from the sync
items = append(items, fstest.Item{
Path: "enormous", Size: 100, ModTime: t1, Md5sum: "8adc5937e635f6c9af646f0b23560fae",
})
err = fs.Sync(flocal, fremote)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
}
// Sync after removing a file and adding a file
func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
// Now delete the local file and download it
func TestCopyAfterDelete(t *testing.T) {
err := os.Remove(localName + "/sub dir/hello world")
if err != nil {
t.Fatalf("Remove failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListingWithPrecision(t, flocal, []fstest.Item{}, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
// Now without dry run
func TestCopy(t *testing.T) {
err := fs.CopyDir(fremote, flocal)
if err != nil {
t.Fatalf("Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
// Sync after changing a file's contents, modtime but not length
func TestSyncAfterChangingContentsOnly(t *testing.T) {
WriteFile("potato", "SMALLER BUT SAME DATE", t2)
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
WriteFile("potato", "smaller but same date", t3)
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
// Create a file and sync it. Change the last modified date and resync.
// If we're only doing sync by size and checksum, we expect nothing to
// to be transferred on the second sync.
func TestSyncBasedOnCheckSum(t *testing.T) {
cleanTempDir(t)
fs.Config.CheckSum = true
defer func() { fs.Config.CheckSum = false }()
WriteFile("check sum", "", t1)
local_items := []fstest.Item{
{Path: "check sum", Size: 0, ModTime: t1, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
fs.Stats.ResetCounters()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Initial sync failed: %v", err)
}
// We should have transferred exactly one file.
if fs.Stats.GetTransfers() != 1 {
t.Fatalf("Sync 1: want 1 transfer, got %d", fs.Stats.GetTransfers())
}
remote_items := local_items
fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
err = os.Chtimes(localName+"/check sum", t2, t2)
if err != nil {
t.Fatalf("Chtimes failed: %v", err)
}
local_items = []fstest.Item{
{Path: "check sum", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
fs.Stats.ResetCounters()
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
// We should have transferred no files
if fs.Stats.GetTransfers() != 0 {
t.Fatalf("Sync 2: want 0 transfers, got %d", fs.Stats.GetTransfers())
}
fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
cleanTempDir(t)
}
// TestFsMkdirRmdirSubdir tests making and removing a sub directory
func TestFsMkdirRmdirSubdir(t *testing.T) {
skipIfNotOk(t)
dir := "dir/subdir"
err := fs.Mkdir(remote, dir)
require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.Config.ModifyWindow)
err = fs.Rmdir(remote, dir)
require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir"}, fs.Config.ModifyWindow)
err = fs.Rmdir(remote, "dir")
require.NoError(t, err)
fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.Config.ModifyWindow)
}
// Create a file and sync it. Change the last modified date and the
// file contents but not the size. If we're only doing sync by size
// only, we expect nothing to to be transferred on the second sync.
func TestSyncSizeOnly(t *testing.T) {
cleanTempDir(t)
fs.Config.SizeOnly = true
defer func() { fs.Config.SizeOnly = false }()
WriteFile("sizeonly", "potato", t1)
local_items := []fstest.Item{
{Path: "sizeonly", Size: 6, ModTime: t1, Md5sum: "8ee2027983915ec78acc45027d874316"},
}
fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
fs.Stats.ResetCounters()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Initial sync failed: %v", err)
}
// We should have transferred exactly one file.
if fs.Stats.GetTransfers() != 1 {
t.Fatalf("Sync 1: want 1 transfer, got %d", fs.Stats.GetTransfers())
}
remote_items := local_items
fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
// Update mtime, md5sum but not length of file
WriteFile("sizeonly", "POTATO", t2)
local_items = []fstest.Item{
{Path: "sizeonly", Size: 6, ModTime: t2, Md5sum: "8ac6f27a282e4938125482607ccfb55f"},
}
fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
fs.Stats.ResetCounters()
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
// We should have transferred no files
if fs.Stats.GetTransfers() != 0 {
t.Fatalf("Sync 2: want 0 transfers, got %d", fs.Stats.GetTransfers())
}
fstest.CheckListingWithPrecision(t, flocal, local_items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, remote_items, fs.Config.ModifyWindow)
cleanTempDir(t)
}
// Test a server side move if possible, or the backup path if not
func TestServerSideMove(t *testing.T) {
fremoteMove, finaliseMove, err := fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
t.Fatalf("Failed to open remote move %q: %v", *RemoteName, err)
}
defer finaliseMove()
t.Logf("Server side move (if possible) %v -> %v", fremote, fremoteMove)
// Start with a copy
err = fs.CopyDir(fremoteMove, fremote)
if err != nil {
t.Fatalf("Server Side Copy failed: %v", err)
}
// Remove one file
obj := fremoteMove.NewFsObject("potato2")
if obj == nil {
t.Fatalf("Failed to find potato2")
}
err = obj.Remove()
if err != nil {
t.Fatalf("Failed to remove object: %v", err)
}
// Do server side move
err = fs.MoveDir(fremoteMove, fremote)
if err != nil {
t.Fatalf("Server Side Move failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, fremote, items[:0], fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremoteMove, items, fs.Config.ModifyWindow)
// Move it back again, dst does not exist this time
err = fs.MoveDir(fremote, fremoteMove)
if err != nil {
t.Fatalf("Server Side Move 2 failed: %v", err)
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremoteMove, items[:0], fs.Config.ModifyWindow)
}
func TestSyncAfterChangingModtimeOnly(t *testing.T) {
WriteFile("empty space", "", t1)
err := os.Chtimes(localName+"/empty space", t2, t2)
if err != nil {
t.Fatalf("Chtimes failed: %v", err)
}
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
// Test a server side copy if possible, or the backup path if not
func TestServerSideCopy(t *testing.T) {
fremoteCopy, finaliseCopy, err := fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
t.Fatalf("Failed to open remote copy %q: %v", *RemoteName, err)
}
defer finaliseCopy()
t.Logf("Server side copy (if possible) %v -> %v", fremote, fremoteCopy)
err = fs.CopyDir(fremoteCopy, fremote)
if err != nil {
t.Fatalf("Server Side Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremoteCopy, items, fs.Config.ModifyWindow)
}
// Test with exclude and delete excluded
func TestSyncWithExcludeAndDeleleteExcluded(t *testing.T) {
fs.Config.Filter.MaxSize = 40
fs.Config.Filter.DeleteExcluded = true
reset := func() {
fs.Config.Filter.MaxSize = 0
fs.Config.Filter.DeleteExcluded = false
}
defer reset()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Check sync the other way round to make sure enormous gets
// deleted even though it is excluded
err = fs.Sync(flocal, fremote)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
// Tidy up - put potato2 back!
reset()
WriteFile("potato2", "------------------------------------------------------------", t1)
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items = []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}
func TestSyncIgnoreExisting(t *testing.T) {
WriteFile("existing", "potato", t1)
fs.Config.IgnoreExisting = true
defer func() { fs.Config.IgnoreExisting = false }()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "existing", Size: 6, ModTime: t1, Md5sum: "8ee2027983915ec78acc45027d874316"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Change everything
WriteFile("existing", "newpotatoes", t2)
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
// Items should not change
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
cleanTempDir(t)
}
// Sync after removing a file and adding a file --dry-run
func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
WriteFile("potato2", "------------------------------------------------------------", t1)
err := os.Remove(localName + "/potato")
if err != nil {
t.Fatalf("Remove failed: %v", err)
}
fs.Config.DryRun = true
err = fs.Sync(fremote, flocal)
fs.Config.DryRun = false
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
before := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
fstest.CheckListingWithPrecision(t, fremote, before, fs.Config.ModifyWindow)
}
// Test with exclude
func TestSyncWithExclude(t *testing.T) {
WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fs.Config.Filter.MaxSize = 80
defer func() {
fs.Config.Filter.MaxSize = 0
}()
err := fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
}