// RandomRemote makes a random bucket or subdirectory on the remote
//
// Call the finalise function returned to Purge the fs at the end (and
// the parent if necessary)
func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) {
var err error
var parentRemote fs.Fs
remoteName, _, err = RandomRemoteName(remoteName)
if err != nil {
return nil, nil, err
}
if subdir {
parentRemote, err = fs.NewFs(remoteName)
if err != nil {
return nil, nil, err
}
remoteName += "/rclone-test-subdir-" + RandomString(8)
}
remote, err := fs.NewFs(remoteName)
if err != nil {
return nil, nil, err
}
finalise := func() {
_ = fs.Purge(remote) // ignore error
if parentRemote != nil {
err = fs.Purge(parentRemote) // ignore error
if err != nil {
log.Printf("Failed to purge %v: %v", parentRemote, err)
}
}
}
return remote, finalise, nil
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, rpath string) (fs.Fs, error) {
mode, err := NewNameEncryptionMode(fs.ConfigFileGet(name, "filename_encryption", "standard"))
if err != nil {
return nil, err
}
password := fs.ConfigFileGet(name, "password", "")
if password == "" {
return nil, errors.New("password not set in config file")
}
password, err = fs.Reveal(password)
if err != nil {
return nil, errors.Wrap(err, "failed to decrypt password")
}
salt := fs.ConfigFileGet(name, "password2", "")
if salt != "" {
salt, err = fs.Reveal(salt)
if err != nil {
return nil, errors.Wrap(err, "failed to decrypt password2")
}
}
cipher, err := newCipher(mode, password, salt)
if err != nil {
return nil, errors.Wrap(err, "failed to make cipher")
}
remote := fs.ConfigFileGet(name, "remote")
if strings.HasPrefix(remote, name+":") {
return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
}
// Look for a file first
remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
wrappedFs, err := fs.NewFs(remotePath)
// if that didn't produce a file, look for a directory
if err != fs.ErrorIsFile {
remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
wrappedFs, err = fs.NewFs(remotePath)
}
if err != fs.ErrorIsFile && err != nil {
return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
}
f := &Fs{
Fs: wrappedFs,
name: name,
root: rpath,
cipher: cipher,
mode: mode,
}
// the features here are ones we could support, and they are
// ANDed with the ones from wrappedFs
f.features = (&fs.Features{
CaseInsensitive: mode == NameEncryptionOff,
DuplicateFiles: true,
ReadMimeType: false, // MimeTypes not supported with crypt
WriteMimeType: false,
}).Fill(f).Mask(wrappedFs)
return f, err
}
func TestInit(t *testing.T) {
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
var err error
fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
t.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
t.Logf("Testing with remote %v", fremote)
localName, err = ioutil.TempDir("", "rclone")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
localName = filepath.ToSlash(localName)
t.Logf("Testing with local %q", localName)
flocal, err = fs.NewFs(localName)
if err != nil {
t.Fatalf("Failed to make %q: %v", remoteName, err)
}
}
// newRun initialise the remote and local for testing and returns a
// run object.
//
// r.flocal is an empty local Fs
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
r := &Run{
Logf: log.Printf,
Fatalf: log.Fatalf,
mkdir: make(map[string]bool),
}
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
var err error
r.fremote, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
r.localName, err = ioutil.TempDir("", "rclone")
if err != nil {
r.Fatalf("Failed to create temp dir: %v", err)
}
r.localName = filepath.ToSlash(r.localName)
r.flocal, err = fs.NewFs(r.localName)
if err != nil {
r.Fatalf("Failed to make %q: %v", r.localName, err)
}
fs.CalculateModifyWindow(r.fremote, r.flocal)
return r
}
// newRun initialise the remote and local for testing and returns a
// run object.
//
// r.flocal is an empty local Fs
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
r := &Run{
Logf: log.Printf,
Fatalf: log.Fatalf,
mkdir: make(map[string]bool),
}
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
*fs.AskPassword = false
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
fs.Config.LowLevelRetries = *LowLevelRetries
var err error
r.fremote, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
r.localName, err = ioutil.TempDir("", "rclone")
if err != nil {
r.Fatalf("Failed to create temp dir: %v", err)
}
r.localName = filepath.ToSlash(r.localName)
r.flocal, err = fs.NewFs(r.localName)
if err != nil {
r.Fatalf("Failed to make %q: %v", r.localName, err)
}
fs.CalculateModifyWindow(r.fremote, r.flocal)
return r
}
func TestInit(t *testing.T) {
var err error
fs.LoadConfig()
fs.Config.Verbose = false
fs.Config.Quiet = true
t.Logf("Using remote %q", RemoteName)
if RemoteName == "" {
RemoteName, err = fstest.LocalRemote()
if err != nil {
log.Fatalf("Failed to create tmp dir: %v", err)
}
}
subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName)
if err != nil {
t.Fatalf("Couldn't make remote name: %v", err)
}
remote, err = fs.NewFs(subRemoteName)
if err == fs.NotFoundInConfigFile {
log.Printf("Didn't find %q in config file - skipping tests", RemoteName)
return
}
if err != nil {
t.Fatalf("Couldn't start FS: %v", err)
}
fstest.TestMkdir(t, remote)
}
// TestInit tests basic intitialisation
func TestInit(t *testing.T) {
var err error
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
*fs.AskPassword = false
fs.LoadConfig()
fs.Config.Verbose = *verbose
fs.Config.Quiet = !*verbose
fs.Config.DumpHeaders = *dumpHeaders
fs.Config.DumpBodies = *dumpBodies
t.Logf("Using remote %q", RemoteName)
if RemoteName == "" {
RemoteName, err = fstest.LocalRemote()
require.NoError(t, err)
}
subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName)
require.NoError(t, err)
remote, err = fs.NewFs(subRemoteName)
if err == fs.ErrorNotFoundInConfigFile {
t.Logf("Didn't find %q in config file - skipping tests", RemoteName)
return
}
require.NoError(t, err)
fstest.TestMkdir(t, remote)
}
// TestFsIsFileNotFound tests that an error is not returned if no object is found
func TestFsIsFileNotFound(t *testing.T) {
skipIfNotOk(t)
remoteName := subRemoteName + "/not found.txt"
fileRemote, err := fs.NewFs(remoteName)
require.NoError(t, err)
fstest.CheckListing(t, fileRemote, []fstest.Item{})
}
// Test a server side move with overlap
func TestServerSideMoveOverlap(t *testing.T) {
r := NewRun(t)
defer r.Finalise()
if r.fremote.Features().DirMove != nil {
t.Skip("Skipping test as remote supports DirMove")
}
subRemoteName := r.fremoteName + "/rclone-move-test"
fremoteMove, err := fs.NewFs(subRemoteName)
require.NoError(t, err)
file1 := r.WriteObject("potato2", "------------------------------------------------------------", t1)
fstest.CheckItems(t, r.fremote, file1)
// Subdir move with no filters should return ErrorCantMoveOverlapping
err = fs.MoveDir(fremoteMove, r.fremote)
assert.EqualError(t, err, fs.ErrorCantMoveOverlapping.Error())
// Now try with a filter which should also fail with ErrorCantMoveOverlapping
fs.Config.Filter.MinSize = 40
defer func() {
fs.Config.Filter.MinSize = -1
}()
err = fs.MoveDir(fremoteMove, r.fremote)
assert.EqualError(t, err, fs.ErrorCantMoveOverlapping.Error())
}
// TestFsListDirRoot tests that DirList works in the root
func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
require.NoError(t, err)
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote, "").GetDirs()
require.NoError(t, err)
assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
}
// newFsDst creates a dst Fs from a name
//
// This must point to a directory
func newFsDst(remote string) fs.Fs {
f, err := fs.NewFs(remote)
if err != nil {
fs.Stats.Error()
log.Fatalf("Failed to create file system for %q: %v", remote, err)
}
return f
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, rpath string) (fs.Fs, error) {
mode, err := NewNameEncryptionMode(fs.ConfigFile.MustValue(name, "filename_encryption", "standard"))
if err != nil {
return nil, err
}
password := fs.ConfigFile.MustValue(name, "password", "")
if password == "" {
return nil, errors.New("password not set in config file")
}
password, err = fs.Reveal(password)
if err != nil {
return nil, errors.Wrap(err, "failed to decrypt password")
}
salt := fs.ConfigFile.MustValue(name, "password2", "")
if salt != "" {
salt, err = fs.Reveal(salt)
if err != nil {
return nil, errors.Wrap(err, "failed to decrypt password2")
}
}
cipher, err := newCipher(mode, password, salt)
if err != nil {
return nil, errors.Wrap(err, "failed to make cipher")
}
remote := fs.ConfigFile.MustValue(name, "remote")
// Look for a file first
remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
wrappedFs, err := fs.NewFs(remotePath)
// if that didn't produce a file, look for a directory
if err != fs.ErrorIsFile {
remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
wrappedFs, err = fs.NewFs(remotePath)
}
if err != fs.ErrorIsFile && err != nil {
return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
}
f := &Fs{
Fs: wrappedFs,
cipher: cipher,
mode: mode,
name: name,
root: rpath,
}
return f, err
}
// TestFsIsFile tests that an error is returned along with a valid fs
// which points to the parent directory.
func TestFsIsFile(t *testing.T) {
skipIfNotOk(t)
remoteName := subRemoteName + "/" + file2.Path
file2Copy := file2
file2Copy.Path = "z.txt"
fileRemote, err := fs.NewFs(remoteName)
assert.Equal(t, fs.ErrorIsFile, err)
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
}
// TestFsListDirRoot tests that DirList works in the root
func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote, "").GetDirs()
require.NoError(t, err)
assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
}
// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
f, err := fs.NewFs(t.remote)
if err != nil {
return err
}
for dir := range f.ListDir() {
if fstest.MatchTestRemote.MatchString(dir.Name) {
log.Printf("Purging %s%s", t.remote, dir.Name)
dir, err := fs.NewFs(t.remote + dir.Name)
if err != nil {
return err
}
err = fs.Purge(dir)
if err != nil {
return err
}
}
}
return nil
}
// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
f, err := fs.NewFs(t.remote)
if err != nil {
return err
}
dirs, err := fs.NewLister().SetLevel(1).Start(f, "").GetDirs()
for _, dir := range dirs {
if fstest.MatchTestRemote.MatchString(dir.Name) {
log.Printf("Purging %s%s", t.remote, dir.Name)
dir, err := fs.NewFs(t.remote + dir.Name)
if err != nil {
return err
}
err = fs.Purge(dir)
if err != nil {
return err
}
}
}
return nil
}
// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
f, err := fs.NewFs(t.remote)
if err != nil {
return err
}
for dir := range f.ListDir() {
insideDigits := len(findInteriorDigits.FindAllString(dir.Name, -1))
if matchTestRemote.MatchString(dir.Name) && insideDigits >= 2 {
log.Printf("Purging %s%s", t.remote, dir.Name)
dir, err := fs.NewFs(t.remote + dir.Name)
if err != nil {
return err
}
err = fs.Purge(dir)
if err != nil {
return err
}
}
}
return nil
}
func TestLimitedFsNotFound(t *testing.T) {
skipIfNotOk(t)
remoteName := subRemoteName + "/not found.txt"
fileRemote, err := fs.NewFs(remoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", remoteName, err)
}
fstest.CheckListing(t, fileRemote, []fstest.Item{})
_, ok := fileRemote.(*fs.Limited)
if ok {
t.Errorf("%v is is a fs.Limited", fileRemote)
}
}
func TestLimitedFs(t *testing.T) {
skipIfNotOk(t)
remoteName := subRemoteName + "/" + file2.Path
file2Copy := file2
file2Copy.Path = "z.txt"
fileRemote, err := fs.NewFs(remoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", remoteName, err)
}
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
_, ok := fileRemote.(*fs.Limited)
if !ok {
t.Errorf("%v is not a fs.Limited", fileRemote)
}
}
func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
found := false
for obj := range rootRemote.ListDir() {
if obj.Name == subRemoteLeaf {
found = true
}
}
if !found {
t.Errorf("Didn't find %q", subRemoteLeaf)
}
}
// TestLimitedFs tests that a LimitedFs is created
func TestLimitedFs(t *testing.T) {
skipIfNotOk(t)
remoteName := subRemoteName + "/" + file2.Path
file2Copy := file2
file2Copy.Path = "z.txt"
fileRemote, err := fs.NewFs(remoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", remoteName, err)
}
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
_, ok := fileRemote.(*fs.Limited)
if !ok {
// Check to see if this wraps a Limited FS
if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap {
_, ok = unwrap.UnWrap().(*fs.Limited)
}
if !ok {
t.Errorf("%v is not a fs.Limited", fileRemote)
}
}
}
// Test with BackupDir set
func TestSyncBackupDir(t *testing.T) {
r := NewRun(t)
defer r.Finalise()
if !fs.CanServerSideMove(r.fremote) {
t.Skip("Skipping test as remote does not support server side move")
}
r.Mkdir(r.fremote)
fs.Config.BackupDir = r.fremoteName + "/backup"
defer func() {
fs.Config.BackupDir = ""
}()
file1 := r.WriteObject("dst/one", "one", t1)
file2 := r.WriteObject("dst/two", "two", t2)
file3 := r.WriteObject("dst/three", "three", t3)
file2a := r.WriteFile("two", "two", t2)
file1a := r.WriteFile("one", "oneone", t2)
fstest.CheckItems(t, r.fremote, file1, file2, file3)
fstest.CheckItems(t, r.flocal, file1a, file2a)
fdst, err := fs.NewFs(r.fremoteName + "/dst")
require.NoError(t, err)
fs.Stats.ResetCounters()
err = fs.Sync(fdst, r.flocal)
require.NoError(t, err)
// file1 is overwritten and the old version moved to backup-dir
file1.Path = "backup/one"
file1a.Path = "dst/one"
// file 2 is unchanged
// file 3 is deleted (moved to backup dir)
file3.Path = "backup/three"
fstest.CheckItems(t, r.fremote, file1, file2, file3, file1a)
}
// TestFsListRoot tests List works in the root
func TestFsListRoot(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
// Should either find file1 and file2 or nothing
found1 := false
f1 := subRemoteLeaf + "/" + file1.Path
found2 := false
f2 := subRemoteLeaf + "/" + file2.Path
f2Alt := subRemoteLeaf + "/" + file2.WinPath
count := 0
errors := fs.Stats.GetErrors()
for obj := range rootRemote.List() {
count++
if obj.Remote() == f1 {
found1 = true
}
if obj.Remote() == f2 || obj.Remote() == f2Alt {
found2 = true
}
}
errors -= fs.Stats.GetErrors()
if count == 0 {
if errors == 0 {
t.Error("Expecting error if count==0")
}
return
}
if found1 && found2 {
if errors != 0 {
t.Error("Not expecting error if found")
}
return
}
t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count)
}