func TestKlientRunningRepairRepair(t *testing.T) {
Convey("", t, func() {
fakeCommandRun := &testutil.FakeCommandRun{}
// 999 is just a randomly chosen port.
klientAddress := "http://127.0.0.1:999/kite"
r := &KlientRunningRepair{
Stdout: util.NewFprint(ioutil.Discard),
KlientService: &klient.KlientService{
KlientAddress: klientAddress,
PauseInterval: time.Millisecond,
MaxAttempts: 5,
},
KlientOptions: klient.KlientOptions{
Address: klientAddress,
Name: "client",
Version: "0.0.0",
},
Exec: fakeCommandRun,
}
Convey("It should call for klient to restart", func() {
r.Repair()
So(fakeCommandRun.RunLog, ShouldResemble, [][]string{
{"sudo", "kd", "restart"},
})
})
Convey("It should fail if status still fails", func() {
err := r.Repair()
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "not-okay")
})
})
}
func NewErrorCommand(stdout io.Writer, log logging.Logger, err error, msg string) *ErrorCommand {
return &ErrorCommand{
Stdout: util.NewFprint(stdout),
Log: log.New("errorCommand"),
Message: msg,
Error: err,
}
}
func NewCommand(i Init, o Options) (*Command, error) {
if err := i.CheckValid(); err != nil {
return nil, err
}
c := &Command{
Options: o,
Log: i.Log,
Stdout: util.NewFprint(i.Stdout),
Helper: i.Helper,
}
return c, nil
}
func TestMountExistsRepair(t *testing.T) {
Convey("Given fs mount and klient mount exist", t, func() {
r := &MountExistsRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: &testutil.FakeKlient{},
Mountcli: &testutil.FakeMountcli{ReturnMountByPath: "foo/bar"},
}
Convey("When Status is run", func() {
Convey("It should okay", func() {
ok, err := r.Status()
So(ok, ShouldBeTrue)
So(err, ShouldBeNil)
})
})
})
Convey("Given klient mount exists but fs mount does not", t, func() {
r := &MountExistsRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: &testutil.FakeKlient{},
Mountcli: &testutil.FakeMountcli{
ReturnMountByPathErr: mountcli.ErrNoMountName,
},
}
Convey("When Status is run", func() {
Convey("It should return an error", func() {
ok, err := r.Status()
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
})
})
})
}
func NewCommand(i Init, o Options) (*Command, error) {
if err := i.CheckValid(); err != nil {
return nil, err
}
if o.Debug {
i.Log.SetLevel(logging.DEBUG)
}
c := &Command{
Init: i,
Options: o,
// Override the init stdout writer with an Fprint writer
Stdout: util.NewFprint(i.Stdout),
}
return c, nil
}
// initSetupRepairers handles creating the repairers that Command needs to operate,
// namely dealing with starting klient, or requirements for klient to run.
//
// If klient isn't able to run, we can't run most of our repairers anyway - that's
// what this one deals with.
func (c *Command) initSetupRepairers() error {
if c.SetupRepairers != nil {
return nil
}
// Our internet repairer retries status many times, until it finally gives up.
internetRepair := &InternetRepair{
Stdout: c.Stdout,
InternetConfirmAddrs: DefaultInternetConfirmAddrs,
HTTPTimeout: time.Second,
RetryOpts: RetryOptions{
StatusRetries: 900,
StatusDelay: 1 * time.Second,
},
}
// The klient running repairer, will check if klient is running and connectable,
// and restart it if not.
klientRunningRepair := &KlientRunningRepair{
Stdout: util.NewFprint(c.Stdout),
KlientOptions: c.KlientOptions,
KlientService: c.KlientService,
Exec: &exec.CommandRun{
Stdin: c.Stdin,
Stdout: c.Stdout,
},
}
// A collection of Repairers responsible for actually repairing a given mount.
// Executed in the order they are defined, the effectiveness of the Repairers
// may depend on the order they are run in.
// *Order matters*
c.SetupRepairers = []Repairer{
internetRepair,
klientRunningRepair,
}
return nil
}
func TestWriteReadRepair(t *testing.T) {
Convey("Given a WriteReadRepair", t, func() {
mountDir, err := ioutil.TempDir("", "writereadrepair")
So(err, ShouldBeNil)
defer os.RemoveAll(mountDir)
fakeKlient := &testutil.FakeKlient{}
r := &WriteReadRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: fakeKlient,
}
Convey("When a directory that can be written to", func() {
fakeKlient.ReturnMountInfo = req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: mountDir,
},
}
fakeKlient.ReturnRemoteExec = command.Output{Stdout: testFileContent}
Convey("When Status() is called", func() {
Convey("It should return okay", func() {
ok, err := r.Status()
So(err, ShouldBeNil)
So(ok, ShouldBeTrue)
})
Convey("It should not remove the MountDir", func() {
r.Status()
exists, err := doesDirExists(mountDir)
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
})
Convey("It should cleanup the test dir after it's done", func() {
r.Status()
// The mount dir should be empty, just like it started with
empty, err := isDirEmpty(mountDir)
So(err, ShouldBeNil)
So(empty, ShouldBeTrue)
})
})
})
Convey("When a directory that cannot be written to", func() {
notExistDir := filepath.Join(mountDir, "i", "dont", "exist")
fakeKlient.ReturnMountInfo = req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: notExistDir,
},
}
Convey("When Status() is called", func() {
Convey("It should return not-okay", func() {
ok, err := r.Status()
So(err, ShouldBeNil)
So(ok, ShouldBeFalse)
})
Convey("It should not remove the MountDir", func() {
r.Status()
exists, err := doesDirExists(mountDir)
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
})
Convey("It should cleanup the test dir after it's done", func() {
r.Status()
// The mount dir should be empty, just like it started with
empty, err := isDirEmpty(mountDir)
So(err, ShouldBeNil)
So(empty, ShouldBeTrue)
})
})
Convey("When Repair() is called", func() {
Convey("It should call RemoteRemount", func() {
r.Repair()
So(fakeKlient.GetCallCount("RemoteRemount"), ShouldEqual, 1)
})
Convey("It should run status again", func() {
r.Repair()
// Checking if Status was called is a bit difficult, so we are
// instead checking if MountInfo is called - which is only called
// from status at the moment.
So(fakeKlient.GetCallCount("RemoteMountInfo"), ShouldEqual, 1)
})
})
})
Convey("When the remote file does not exist", func() {
fakeKlient.ReturnMountInfo = req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: mountDir,
},
}
fakeKlient.ReturnRemoteExec = command.Output{ExitStatus: 1}
Convey("When Status() is called", func() {
Convey("It should return not okay", func() {
ok, err := r.Status()
So(err, ShouldBeNil)
So(ok, ShouldBeFalse)
})
Convey("It should not remove the MountDir", func() {
r.Status()
exists, err := doesDirExists(mountDir)
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
})
Convey("It should cleanup the test dir after it's done", func() {
r.Status()
// The mount dir should be empty, just like it started with
empty, err := isDirEmpty(mountDir)
So(err, ShouldBeNil)
So(empty, ShouldBeTrue)
})
})
})
Convey("When the remote files contents do not match", func() {
fakeKlient.ReturnMountInfo = req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: mountDir,
},
}
fakeKlient.ReturnRemoteExec = command.Output{Stdout: "badcontents"}
Convey("When Status() is called", func() {
Convey("It should return not okay", func() {
ok, err := r.Status()
So(err, ShouldBeNil)
So(ok, ShouldBeFalse)
})
Convey("It should not remove the MountDir", func() {
r.Status()
exists, err := doesDirExists(mountDir)
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
})
Convey("It should cleanup the test dir after it's done", func() {
r.Status()
// The mount dir should be empty, just like it started with
empty, err := isDirEmpty(mountDir)
So(err, ShouldBeNil)
So(empty, ShouldBeTrue)
})
})
})
})
}
func TestKlientRunningRepairStatus(t *testing.T) {
Convey("Given a running klient", t, func() {
s := kite.New("server", "0.0.0")
s.Config.DisableAuthentication = true
ts := httptest.NewServer(s)
klientAddress := fmt.Sprintf("%s/kite", ts.URL)
r := &KlientRunningRepair{
Stdout: util.NewFprint(ioutil.Discard),
KlientService: &klient.KlientService{
KlientAddress: klientAddress,
PauseInterval: time.Millisecond,
MaxAttempts: 5,
},
KlientOptions: klient.KlientOptions{
Address: klientAddress,
Name: "client",
Version: "0.0.0",
},
}
Convey("It should show status ok", func() {
ok, err := r.Status()
So(ok, ShouldBeTrue)
So(err, ShouldBeNil)
})
})
Convey("Given klient running, but not dial-able", t, func() {
// This web server responds like a kite, but can't be dialed
ts := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "Welcome to SockJS!\n")
}))
defer ts.Close()
klientAddress := fmt.Sprintf("%s/kite", ts.URL)
r := &KlientRunningRepair{
Stdout: util.NewFprint(ioutil.Discard),
KlientService: &klient.KlientService{
KlientAddress: klientAddress,
PauseInterval: time.Millisecond,
MaxAttempts: 5,
},
KlientOptions: klient.KlientOptions{
Address: klientAddress,
Name: "client",
Version: "0.0.0",
},
}
Convey("It should show status not ok", func() {
ok, err := r.Status()
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
})
})
Convey("Given a not running klient", t, func() {
// 999 is just a randomly chosen port.
klientAddress := "http://127.0.0.1:999/kite"
r := &KlientRunningRepair{
Stdout: util.NewFprint(ioutil.Discard),
KlientService: &klient.KlientService{
KlientAddress: klientAddress,
PauseInterval: time.Millisecond,
MaxAttempts: 5,
},
KlientOptions: klient.KlientOptions{
Address: klientAddress,
Name: "client",
Version: "0.0.0",
},
}
Convey("It should show status not ok", func() {
ok, err := r.Status()
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
})
})
}
func TestPermDeniedRepair(t *testing.T) {
Convey("Given the mount dir has 655 perms", t, func() {
tmpDir, err := ioutil.TempDir("", "permdeniedrepair")
So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir)
permDir := filepath.Join(tmpDir, "dir")
// Make the perm denied dir.. as best we can replicate.
err = os.Mkdir(permDir, 0655)
So(err, ShouldBeNil)
fakeKlient := &testutil.FakeKlient{
ReturnMountInfo: req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: permDir,
},
},
}
r := &PermDeniedRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: fakeKlient,
}
Convey("When Status is run", func() {
Convey("It should return not okay", func() {
ok, err := r.Status()
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
})
})
Convey("When Repair is run", func() {
Convey("It should call RemoteRemount", func() {
// Ignoring error here, because Repair will simply return that the dir
// still has 655. Because it does.
r.Repair()
So(fakeKlient.GetCallCount("RemoteRemount"), ShouldEqual, 1)
})
Convey("It should check the status of the mount again", func() {
err := r.Repair()
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "not-okay")
})
})
})
Convey("Given the mount dir does not have 655 perms", t, func() {
tmpDir, err := ioutil.TempDir("", "permdeniedrepair")
So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir)
permDir := filepath.Join(tmpDir, "dir")
// Make the perm denied dir.. as best we can replicate.
err = os.Mkdir(permDir, 0755)
So(err, ShouldBeNil)
fakeKlient := &testutil.FakeKlient{
ReturnMountInfo: req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: permDir,
},
},
}
r := &PermDeniedRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: fakeKlient,
}
Convey("When Status is run", func() {
Convey("It should return okay", func() {
ok, err := r.Status()
So(ok, ShouldBeTrue)
So(err, ShouldBeNil)
})
})
})
}
// initDefaultRepairers creates the repairers for this Command if the
// Command.Repairers field is *nil*. This allows a caller can specify their own
// repairers if desired.
func (c *Command) initDefaultRepairers() error {
if c.Repairers != nil {
return nil
}
// TODO: Re-enable. Currently disabled because we're manually running it
// before the checkMachineExist() call inside of Run().
//
//// The kontrol repairer will check if we're connected to kontrol yet, and
//// attempt to wait for it. Eventually restarting if needed.
//kontrolRepair := &KontrolRepair{
// Log: c.Log.New("KontrolRepair"),
// Stdout: c.Stdout,
// Klient: c.Klient,
// RetryOptions: RetryOptions{
// StatusRetries: 3,
// StatusDelay: 10 * time.Second,
// },
// Exec: &exec.CommandRun{
// Stdin: c.Stdin,
// Stdout: c.Stdout,
// },
//}
// The kite unreachable repairer ensures that the remote machine is on, and
// kite is reachable. No repair action is possible.
kiteUnreachableRepair := &KiteUnreachableRepair{
Log: c.Log.New("KiteUnreachableRepair"),
Stdout: c.Stdout,
Klient: c.Klient,
StatusRetries: 10,
StatusDelay: 1 * time.Second,
MachineName: c.Options.MountName,
}
// The token expired repair checks for token expired. This should be placed *before*
// TokenNotYetValidRepair, so that after we restart, we can check if the token
// is valid.
tokenExpired := &TokenExpiredRepair{
Log: c.Log.New("TokenExpiredRepair"),
Stdout: c.Stdout,
Klient: c.Klient,
RepairWaitForToken: 5 * time.Second,
MachineName: c.Options.MountName,
}
// The token not yet valid repairer will check if we're failing from the token
// not yet valid error, and wait for it to become valid.
tokenNotValidYetRepair := &TokenNotYetValidRepair{
Log: c.Log.New("TokenNotYetValidRepair"),
Stdout: c.Stdout,
Klient: c.Klient,
RepairRetries: 5,
RepairDelay: 3 * time.Second,
MachineName: c.Options.MountName,
}
mountExistsRepair := &MountExistsRepair{
Log: c.Log.New("MountExistsRepair"),
Stdout: util.NewFprint(c.Stdout),
MountName: c.Options.MountName,
Klient: c.Klient,
Mountcli: mountcli.NewMountcli(),
}
permDeniedRepair := &PermDeniedRepair{
Log: c.Log.New("PermDeniedRepair"),
Stdout: util.NewFprint(c.Stdout),
MountName: c.Options.MountName,
Klient: c.Klient,
}
mountEmptyRepair := &MountEmptyRepair{
Log: c.Log.New("MountEmptyRepair"),
Stdout: util.NewFprint(c.Stdout),
MountName: c.Options.MountName,
Klient: c.Klient,
}
deviceNotConfiguredRepair := &DeviceNotConfiguredRepair{
Log: c.Log.New("DeviceNotConfiguredRepair"),
Stdout: util.NewFprint(c.Stdout),
MountName: c.Options.MountName,
Klient: c.Klient,
}
writeReadRepair := &WriteReadRepair{
Log: c.Log.New("WriteReadRepair"),
Stdout: util.NewFprint(c.Stdout),
MountName: c.Options.MountName,
Klient: c.Klient,
}
// A collection of Repairers responsible for actually repairing a given mount.
// Executed in the order they are defined, the effectiveness of the Repairers
// may depend on the order they are run in. An example being TokenNotValidYetRepair
// likely should be run *after* a restart, as Tokens not being valid yet usually
// happens after a restart.
c.Repairers = []Repairer{
//kontrolRepair,
kiteUnreachableRepair,
tokenExpired,
tokenNotValidYetRepair,
mountExistsRepair,
permDeniedRepair,
mountEmptyRepair,
deviceNotConfiguredRepair,
writeReadRepair,
}
return nil
}
func TestMountEmptyRepair(t *testing.T) {
Convey("Given the mount path is empty", t, func() {
tmpDir, err := ioutil.TempDir("", "mountemptyrepair")
So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir)
fakeKlient := &testutil.FakeKlient{
ReturnMountInfo: req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: tmpDir,
},
},
}
r := &MountEmptyRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: fakeKlient,
}
Convey("When Status is run", func() {
Convey("It should be not okay", func() {
ok, err := r.Status()
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
})
})
Convey("When Repair is run", func() {
Convey("It should call RemoteRemount", func() {
// Ignoring error here, because Repair will simply return that the dir
// is still empty. Because it is.
r.Repair()
So(fakeKlient.GetCallCount("RemoteRemount"), ShouldEqual, 1)
})
Convey("It should check the status of the mount again", func() {
err := r.Repair()
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "not-okay")
})
})
})
Convey("Given the mount path is not empty", t, func() {
tmpDir, err := ioutil.TempDir("", "mountemptyrepair")
So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir)
fakeKlient := &testutil.FakeKlient{
ReturnMountInfo: req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: tmpDir,
},
},
}
r := &MountEmptyRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: fakeKlient,
}
Convey("With a file", func() {
f, err := os.Create(filepath.Join(tmpDir, "file"))
So(err, ShouldBeNil)
f.Close()
Convey("When Status is run", func() {
Convey("It should return okay", func() {
ok, err := r.Status()
So(ok, ShouldBeTrue)
So(err, ShouldBeNil)
})
})
})
Convey("With a dir", func() {
So(os.Mkdir(filepath.Join(tmpDir, "dir"), 0755), ShouldBeNil)
Convey("When Status is run", func() {
Convey("It should not return an error", func() {
ok, err := r.Status()
So(ok, ShouldBeTrue)
So(err, ShouldBeNil)
})
})
})
})
}
func TestDeviceNotConfiguredRepair(t *testing.T) {
SkipConvey("", t, func() {
tmpDir, err := ioutil.TempDir("", "devicenotconfiguredrepair")
So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir)
mountDir := filepath.Join(tmpDir, "mount")
So(os.Mkdir(mountDir, 0755), ShouldBeNil)
fakeKlient := &testutil.FakeKlient{
ReturnMountInfo: req.MountInfoResponse{
MountFolder: req.MountFolder{
LocalPath: mountDir,
},
},
}
r := &DeviceNotConfiguredRepair{
Log: discardLogger,
Stdout: util.NewFprint(ioutil.Discard),
Klient: fakeKlient,
}
Convey("Given a normal dir", func() {
Convey("When Status() is called", func() {
Convey("It should return okay", func() {
ok, err := r.Status()
So(err, ShouldBeNil)
So(ok, ShouldBeTrue)
})
})
})
Convey("Given a DeviceNotConfigured dir", func() {
err = MakeDirDeviceNotConfigured(mountDir)
So(err, ShouldBeNil)
defer fuseklient.Unmount(mountDir)
Convey("When Status() is called", func() {
Convey("It should return not okay", func() {
ok, err := r.Status()
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
})
})
Convey("When Repair() is called", func() {
Convey("It should call RemoteRemount", func() {
// Ignoring the error, because it's going to return the status error anyway.
r.Repair()
So(fakeKlient.GetCallCount("RemoteRemount"), ShouldEqual, 1)
})
Convey("It should run status again", func() {
err := r.Repair()
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "not-okay")
})
})
})
})
}