// initConfig is run by cobra after initialising the flags
func initConfig() {
// Log file output
if *logFile != "" {
f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
if err != nil {
log.Fatalf("Failed to open log file: %v", err)
}
_, err = f.Seek(0, os.SEEK_END)
if err != nil {
fs.ErrorLog(nil, "Failed to seek log file to end: %v", err)
}
log.SetOutput(f)
fs.DebugLogger.SetOutput(f)
redirectStderr(f)
}
// Load the rest of the config now we have started the logger
fs.LoadConfig()
// Write the args for debug purposes
fs.Debug("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
// Setup CPU profiling if desired
if *cpuProfile != "" {
fs.Log(nil, "Creating CPU profile %q\n", *cpuProfile)
f, err := os.Create(*cpuProfile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
// Setup memory profiling if desired
if *memProfile != "" {
defer func() {
fs.Log(nil, "Saving Memory profile %q\n", *memProfile)
f, err := os.Create(*memProfile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.WriteHeapProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = f.Close()
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
}()
}
}
// newRun initialise the remote and local for testing and returns a
// run object.
//
// r.flocal is an empty local Fs
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
r := &Run{
Logf: log.Printf,
Fatalf: log.Fatalf,
mkdir: make(map[string]bool),
}
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
var err error
r.fremote, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
r.localName, err = ioutil.TempDir("", "rclone")
if err != nil {
r.Fatalf("Failed to create temp dir: %v", err)
}
r.localName = filepath.ToSlash(r.localName)
r.flocal, err = fs.NewFs(r.localName)
if err != nil {
r.Fatalf("Failed to make %q: %v", r.localName, err)
}
fs.CalculateModifyWindow(r.fremote, r.flocal)
return r
}
func TestInit(t *testing.T) {
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
var err error
fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
t.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
t.Logf("Testing with remote %v", fremote)
localName, err = ioutil.TempDir("", "rclone")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
localName = filepath.ToSlash(localName)
t.Logf("Testing with local %q", localName)
flocal, err = fs.NewFs(localName)
if err != nil {
t.Fatalf("Failed to make %q: %v", remoteName, err)
}
}
// TestInit tests basic intitialisation
func TestInit(t *testing.T) {
var err error
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
*fs.AskPassword = false
fs.LoadConfig()
fs.Config.Verbose = *verbose
fs.Config.Quiet = !*verbose
fs.Config.DumpHeaders = *dumpHeaders
fs.Config.DumpBodies = *dumpBodies
t.Logf("Using remote %q", RemoteName)
if RemoteName == "" {
RemoteName, err = fstest.LocalRemote()
require.NoError(t, err)
}
subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName)
require.NoError(t, err)
remote, err = fs.NewFs(subRemoteName)
if err == fs.ErrorNotFoundInConfigFile {
t.Logf("Didn't find %q in config file - skipping tests", RemoteName)
return
}
require.NoError(t, err)
fstest.TestMkdir(t, remote)
}
func TestInit(t *testing.T) {
var err error
fs.LoadConfig()
fs.Config.Verbose = false
fs.Config.Quiet = true
t.Logf("Using remote %q", RemoteName)
if RemoteName == "" {
RemoteName, err = fstest.LocalRemote()
if err != nil {
log.Fatalf("Failed to create tmp dir: %v", err)
}
}
subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(RemoteName)
if err != nil {
t.Fatalf("Couldn't make remote name: %v", err)
}
remote, err = fs.NewFs(subRemoteName)
if err == fs.NotFoundInConfigFile {
log.Printf("Didn't find %q in config file - skipping tests", RemoteName)
return
}
if err != nil {
t.Fatalf("Couldn't start FS: %v", err)
}
fstest.TestMkdir(t, remote)
}
// newRun initialise the remote mount for testing and returns a run
// object.
//
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
r := &Run{
umountResult: make(chan error, 1),
}
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
*fs.AskPassword = false
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
fs.Config.LowLevelRetries = *LowLevelRetries
var err error
r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
log.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
r.mountPath, err = ioutil.TempDir("", "rclonefs-mount")
if err != nil {
log.Fatalf("Failed to create mount dir: %v", err)
}
// Mount it up
r.mount()
return r
}
// newRun initialise the remote and local for testing and returns a
// run object.
//
// r.flocal is an empty local Fs
// r.fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
r := &Run{
Logf: log.Printf,
Fatalf: log.Fatalf,
mkdir: make(map[string]bool),
}
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
*fs.AskPassword = false
fs.LoadConfig()
fs.Config.Verbose = *Verbose
fs.Config.Quiet = !*Verbose
fs.Config.DumpHeaders = *DumpHeaders
fs.Config.DumpBodies = *DumpBodies
fs.Config.LowLevelRetries = *LowLevelRetries
var err error
r.fremote, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
r.localName, err = ioutil.TempDir("", "rclone")
if err != nil {
r.Fatalf("Failed to create temp dir: %v", err)
}
r.localName = filepath.ToSlash(r.localName)
r.flocal, err = fs.NewFs(r.localName)
if err != nil {
r.Fatalf("Failed to make %q: %v", r.localName, err)
}
fs.CalculateModifyWindow(r.fremote, r.flocal)
return r
}
func main() {
flag.Parse()
if *runTests != "" {
remotes = strings.Split(*runTests, ",")
}
log.Printf("Testing remotes: %s", strings.Join(remotes, ", "))
start := time.Now()
if *clean {
fs.LoadConfig()
} else {
makeTestBinary()
defer removeTestBinary()
}
// start the tests
results := make(chan *test, 8)
awaiting := 0
for _, remote := range remotes {
awaiting += 2
go newTest(remote, false).run(results)
go newTest(remote, true).run(results)
}
// Wait for the tests to finish
var failed []*test
for ; awaiting > 0; awaiting-- {
t := <-results
if !t.passed() {
failed = append(failed, t)
}
}
duration := time.Since(start)
// Summarise results
if len(failed) == 0 {
log.Printf("PASS: All tests finished OK in %v", duration)
} else {
log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
for _, t := range failed {
log.Printf(" * %s", t.cmdString)
}
os.Exit(1)
}
}
// ParseFlags parses the command line flags
func ParseFlags() {
pflag.Usage = syntaxError
pflag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
fs.LoadConfig()
// Setup profiling if desired
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
}
// ParseFlags parses the command line flags
func ParseFlags() {
pflag.Usage = syntaxError
pflag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
fs.LoadConfig()
}
func main() {
ParseFlags()
if *version {
fmt.Printf("rclone %s\n", fs.Version)
os.Exit(0)
}
command, args := ParseCommand()
// Log file output
if *logFile != "" {
f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
if err != nil {
log.Fatalf("Failed to open log file: %v", err)
}
_, err = f.Seek(0, os.SEEK_END)
if err != nil {
fs.ErrorLog(nil, "Failed to seek log file to end: %v", err)
}
log.SetOutput(f)
fs.DebugLogger.SetOutput(f)
redirectStderr(f)
}
// Load the rest of the config now we have started the logger
fs.LoadConfig()
// Write the args for debug purposes
fs.Debug("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
// Setup CPU profiling if desired
if *cpuProfile != "" {
fs.Log(nil, "Creating CPU profile %q\n", *cpuProfile)
f, err := os.Create(*cpuProfile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
// Setup memory profiling if desired
if *memProfile != "" {
defer func() {
fs.Log(nil, "Saving Memory profile %q\n", *memProfile)
f, err := os.Create(*memProfile)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = pprof.WriteHeapProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
err = f.Close()
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
}()
}
// Make source and destination fs
var fdst, fsrc fs.Fs
if len(args) >= 1 {
fdst = NewFsSrc(args[0])
}
if len(args) >= 2 {
fsrc = fdst
fdst = NewFs(args[1])
}
fs.CalculateModifyWindow(fdst, fsrc)
if !command.NoStats {
StartStats()
}
// Exit if no command to run
if command.Run == nil {
return
}
// Run the actual command
var err error
for try := 1; try <= *retries; try++ {
err = command.Run(fdst, fsrc)
if !command.Retry || (err == nil && !fs.Stats.Errored()) {
break
}
if fs.IsFatalError(err) {
fs.Log(nil, "Fatal error received - not attempting retries")
break
}
if fs.IsNoRetryError(err) {
fs.Log(nil, "Can't retry this error - not attempting retries")
break
}
if err != nil {
fs.Log(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, fs.Stats.GetErrors(), err)
} else {
fs.Log(nil, "Attempt %d/%d failed with %d errors", try, *retries, fs.Stats.GetErrors())
}
if try < *retries {
fs.Stats.ResetErrors()
}
}
if err != nil {
log.Fatalf("Failed to %s: %v", command.Name, err)
}
if !command.NoStats && (!fs.Config.Quiet || fs.Stats.Errored() || *statsInterval > 0) {
fs.Log(nil, "%s", fs.Stats)
}
if fs.Config.Verbose {
fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
}
if fs.Stats.Errored() {
os.Exit(1)
}
}