// NewRemote creates a new Remote instance.
func NewRemote(opts *RemoteOptions) *Remote {
// TODO: Improve this usage. Basically i want a proper koding/logging struct,
// but we need to create it from somewhere. klient is always uses kite.Logger,
// which **should** always be implemented by a koding/logging.Logger.. but in
// the event that it's not, how do we handle it?
kodingLog, ok := opts.Log.(logging.Logger)
if !ok {
opts.Log.Error(
"Unable to convert koding/kite.Logger to koding/logging.Logger. Creating new logger",
)
kodingLog = logging.NewLogger("new-logger")
}
kodingLog = kodingLog.New("remote")
// Setting the handler to debug, because various Remote methods allow a
// debug option, and this saves us from having to set the handler level every time.
// This only sets handler, not the actual loglevel.
logging.DefaultHandler.SetLevel(logging.DEBUG)
r := &Remote{
localKite: opts.Kite,
kitesGetter: &KodingKite{Kite: opts.Kite},
storage: opts.Storage,
log: kodingLog,
machinesErrCacheMax: 5 * time.Minute,
machinesCacheMax: 10 * time.Second,
machineNamesCache: map[string]string{},
unmountPath: fuseklient.Unmount,
machines: machine.NewMachines(kodingLog, opts.Storage),
maxRestoreAttempts: defaultMaxRestoreAttempts,
restoreFailuresPause: defaultRestoreFailuresPause,
eventSub: opts.EventSub,
}
return r
}
func TestGetMachines(t *testing.T) {
kg := newMockKiteGetter()
kg.AddByUrl("http://testhost1:56789")
store := storage.NewMemoryStorage()
r := &Remote{
localKite: &kite.Kite{
Id: "test id",
Config: &config.Config{
Username: "******",
},
},
kitesGetter: kg,
log: discardLogger,
machinesCacheMax: 1 * time.Second,
machineNamesCache: map[string]string{},
machines: machine.NewMachines(discardLogger, store),
storage: store,
}
machines, err := r.GetMachines()
if err != nil {
t.Error(err)
}
// Should return all kites
if machines.Count() != 1 {
t.Fatalf(
"Expected GetKites to return all current kites. Wanted %d, got %d",
1, machines.Count(),
)
}
kg.AddByUrl("http://testhost2:56789")
machines, err = r.GetMachines()
if err != nil {
t.Error(err)
}
// Should return only the cached kites
if machines.Count() != 1 {
t.Errorf(
"Expected GetKites to cache results. Expected %d, got %d",
1, machines.Count(),
)
}
// Wait longer than the kite timeout
time.Sleep(2 * time.Second)
machines, err = r.GetMachines()
if err != nil {
t.Error(err)
}
// Should not use the cache
if machines.Count() != 2 {
t.Errorf(
"Expected GetKites clear cache. Expected %d results, got %d",
2, machines.Count(),
)
}
// Tell the kite getter to fail
kg.GetKitesError = errors.New("KitesGetter test error")
kg.AddByUrl("http://testhost3:56789")
machines, err = r.GetMachines()
// Should return the old results if the KitesGetter returns an error
if machines.Count() != 2 {
t.Errorf(
"Expected GetKites return the cache when KitesGetter fails, within the clientsErrCacheMax duration. Expected %d results, got %d",
2, machines.Count(),
)
}
// Should not return an error when returning a cache
if err != nil {
t.Errorf("Expected GetKites to not return an error if it is returning a cache. Expected nil, got '%s'", err.Error())
}
// Sleep past the clientsErrCacheMax duration
time.Sleep(3 * time.Second)
machines, err = r.GetMachines()
// Should return the error when the clientsErrCacheMax is too old
if err == nil {
t.Error("Expected GetKites to return an error when after clientsErrCacheMax. Got nil.")
}
// Should not return any results after the clientsErrCacheMax
// duration
if machines != nil {
t.Errorf(
"Expected GetMachines not to return the machines cache when KitesGetter failes after clientsErrCacheMax. Expected nil machine",
)
}
}
func TestMatchMachineToKite(t *testing.T) {
Convey("Given a Remote", t, func() {
kg := newMockKiteGetter()
store := storage.NewMemoryStorage()
r := &Remote{
log: discardLogger,
kitesGetter: kg,
machines: machine.NewMachines(discardLogger, store),
}
Convey("Given a machine with a URL of http://foo.url", func() {
m := &machine.Machine{MachineMeta: machine.MachineMeta{
URL: "http://foo.url",
IP: "foo.url",
}}
So(r.machines.Add(m), ShouldBeNil)
Convey("With a kite for http://foo.url", func() {
kg.AddByUrl("http://foo.url")
Convey("It should match", func() {
match, err := r.matchMachineToKite(kg.Clients[0].Client)
So(err, ShouldBeNil)
So(match, ShouldEqual, m)
})
})
Convey("With a kite for http://foo.url:8080", func() {
kg.AddByUrl("http://foo.url:8080")
Convey("It should not match", func() {
match, err := r.matchMachineToKite(kg.Clients[0].Client)
So(err, ShouldEqual, machine.ErrMachineNotFound)
So(match, ShouldBeNil)
})
})
})
Convey("Given a machine with a blank URL and an IP of foo.url", func() {
m := &machine.Machine{MachineMeta: machine.MachineMeta{
URL: "",
IP: "foo.url",
}}
So(r.machines.Add(m), ShouldBeNil)
Convey("With a kite for http://foo.url", func() {
kg.AddByUrl("http://foo.url")
Convey("It should match", func() {
match, err := r.matchMachineToKite(kg.Clients[0].Client)
So(err, ShouldBeNil)
So(match, ShouldEqual, m)
})
})
Convey("With a kite for http://foo.url:8080", func() {
kg.AddByUrl("http://foo.url:8080")
Convey("It should match", func() {
match, err := r.matchMachineToKite(kg.Clients[0].Client)
So(err, ShouldBeNil)
So(match, ShouldEqual, m)
})
})
})
})
}
func TestGetMachinesWithoutCache(t *testing.T) {
Convey("Given a Remote", t, func() {
kg := newMockKiteGetter()
store := storage.NewMemoryStorage()
r := &Remote{
localKite: &kite.Kite{
Id: "test id",
Config: &config.Config{
Username: "******",
},
},
kitesGetter: kg,
log: discardLogger,
machines: machine.NewMachines(discardLogger, store),
machinesCacheMax: 1 * time.Second,
machineNamesCache: map[string]string{},
storage: store,
}
// Sanity check our count.
So(r.machines.Count(), ShouldEqual, 0)
Convey("Given a new machine", func() {
kg.AddByUrl("http://testhost1:56789")
// Sanity check our config
So(kg.Clients[0].Reconnect, ShouldBeFalse)
Convey("It should configure the kite Client", func() {
r.GetMachinesWithoutCache()
So(kg.Clients[0].Reconnect, ShouldBeTrue)
})
Convey("It should add the new machine", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
So(machines, ShouldNotBeNil)
So(machines.Count(), ShouldEqual, 1)
})
Convey("It should create the HTTPTracker", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
So(machines, ShouldNotBeNil)
mach := machines.Machines()[0]
So(mach, ShouldNotBeNil)
So(mach.HTTPTracker, ShouldNotBeNil)
Convey("It should start the HTTPTracker", func() {
So(mach.HTTPTracker.IsPinging(), ShouldBeTrue)
})
})
})
Convey("Given a loaded but not yet valid machine", func() {
kg.AddByUrl("http://testhost1:56789")
// Add a machine, with just enough info to serve our needs
r.machines.Add(&machine.Machine{
MachineMeta: machine.MachineMeta{
IP: "testhost1",
},
})
// Sanity check our config
So(kg.Clients[0].Reconnect, ShouldBeFalse)
Convey("It should configure the kite Client", func() {
r.GetMachinesWithoutCache()
So(kg.Clients[0].Reconnect, ShouldBeTrue)
})
Convey("It should not add a new machine for the kite", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
So(machines, ShouldNotBeNil)
So(machines.Count(), ShouldEqual, 1)
})
Convey("It should create the HTTPTracker", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
So(machines, ShouldNotBeNil)
mach := machines.Machines()[0]
So(mach, ShouldNotBeNil)
So(mach.HTTPTracker, ShouldNotBeNil)
Convey("It should start the HTTPTracker", func() {
So(mach.HTTPTracker.IsPinging(), ShouldBeTrue)
})
})
})
Convey("Given an pre existing valid machine", func() {
kg.AddByUrl("http://testhost1:56789")
// Add a machine, with just enough info to serve our needs
validMachine := &machine.Machine{
MachineMeta: machine.MachineMeta{
IP: "testhost1",
},
Log: testutil.DiscardLogger,
KiteTracker: kitepinger.NewPingTracker(nil), // Invalid ping trackers, but
HTTPTracker: kitepinger.NewPingTracker(nil), // okay for this test currently
// a bit of a trick, Machine implements Transport, so we're using an empty
// machine here to implement transport, and thus make the validMachine instance
// "valid". Both this and the pingtrackers above are not usable, but satisfy
// the checkvalid requirements.
Transport: &machine.Machine{},
}
So(r.machines.Add(validMachine), ShouldBeNil)
// sanity check, to make sure it's valid
So(validMachine.CheckValid(), ShouldBeNil)
Convey("It should be the same machine instance", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
returnedMachine := machines.Machines()[0]
So(returnedMachine, ShouldEqual, validMachine)
})
Convey("It should only return 1 machine", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
So(machines.Count(), ShouldEqual, 1)
})
Convey("With a label that does not match the local label", func() {
kg.Clients[0].MachineLabel = "foobarbaz"
// Sanity check
So(kg.Clients[0].MachineLabel, ShouldNotEqual, validMachine.MachineLabel)
Convey("It should update the machine label", func() {
machines, err := r.GetMachinesWithoutCache()
So(err, ShouldBeNil)
returnedMachine := machines.Machines()[0]
So(returnedMachine.MachineLabel, ShouldEqual, validMachine.MachineLabel)
})
})
})
})
}
func TestRestoreMounts(t *testing.T) {
Convey("Given a Remote", t, func() {
callCounts := map[string]int{}
callOrder := []string{}
kg := newMockKiteGetter()
store := storage.NewMemoryStorage()
r := &Remote{
localKite: &kite.Kite{
Id: "test id",
Config: &config.Config{
Username: "******",
},
},
kitesGetter: kg,
log: discardLogger,
machines: machine.NewMachines(discardLogger, store),
machinesCacheMax: 1 * time.Second,
machineNamesCache: map[string]string{},
storage: store,
// Set it to a default value, so we don't accidentally call real method
mockedRestoreMount: func(m *mount.Mount) error {
return errors.New("default error")
},
}
Convey("With some machines associated to the mounts", func() {
r.mounts = []*mount.Mount{
{IP: "foo"},
{IP: "bar"},
}
r.maxRestoreAttempts = 3
kg.AddByUrl("http://foo/kite")
kg.AddByUrl("http://bar/kite")
kg.AddByUrl("http://baz/kite")
Convey("It should set machines with mounts' status to remounting", func() {
r.restoreMounts()
m, err := r.machines.GetByIP("foo")
So(err, ShouldBeNil)
status, statusMsg := m.GetRawStatus()
So(status, ShouldEqual, machine.MachineRemounting)
So(statusMsg, ShouldEqual, autoRemounting)
m, err = r.machines.GetByIP("bar")
So(err, ShouldBeNil)
status, statusMsg = m.GetRawStatus()
So(status, ShouldEqual, machine.MachineRemounting)
So(statusMsg, ShouldEqual, autoRemounting)
})
Convey("It should not set machines without mounts' status", func() {
r.restoreMounts()
m, err := r.machines.GetByIP("baz")
So(err, ShouldBeNil)
status, statusMsg := m.GetRawStatus()
So(status, ShouldEqual, machine.MachineStatusUnknown)
So(statusMsg, ShouldEqual, "")
})
})
Convey("With a mount that succeeds", func() {
r.mounts = []*mount.Mount{{IP: "foo"}}
r.maxRestoreAttempts = 5
r.mockedRestoreMount = func(m *mount.Mount) error {
callCounts[m.IP]++
return nil
}
Convey("It should restore it only once", func() {
So(r.restoreMounts(), ShouldBeNil)
So(callCounts, ShouldResemble, map[string]int{"foo": 1})
})
Convey("It should not remove the mount from the internal mount slice", func() {
So(r.restoreMounts(), ShouldBeNil)
So(len(r.mounts), ShouldEqual, 1)
})
})
Convey("With a mount that succeeds and another that fails", func() {
fakeErr := errors.New("Fake error")
r.mounts = []*mount.Mount{
{IP: "foo"},
{IP: "bar"},
}
r.maxRestoreAttempts = 4
r.mockedRestoreMount = func(m *mount.Mount) error {
callCounts[m.IP]++
callOrder = append(callOrder, m.IP)
if m.IP == "bar" {
return fakeErr
}
return nil
}
Convey("It should return the error after giving max attempts", func() {
So(r.restoreMounts(), ShouldEqual, fakeErr)
})
Convey("It should only attempt the successful mount once", func() {
r.restoreMounts()
calls, ok := callCounts["foo"]
So(ok, ShouldBeTrue) // prevent panics
So(calls, ShouldEqual, 1)
})
Convey("It should repeat attempts for the failure", func() {
r.restoreMounts()
calls, ok := callCounts["bar"]
So(ok, ShouldBeTrue) // prevent panics
So(calls, ShouldEqual, 3)
})
Convey("It should call in the expected order", func() {
r.restoreMounts()
So(callOrder, ShouldResemble, []string{
"foo",
"bar",
"bar",
"bar",
})
})
})
Convey("With a mount that eventually succeeds", func() {
r.mounts = []*mount.Mount{
{IP: "foo"},
{IP: "bar"},
{IP: "baz"},
}
r.maxRestoreAttempts = 20 // more than we should need, given working code.
r.mockedRestoreMount = func(m *mount.Mount) error {
callCounts[m.IP]++
callOrder = append(callOrder, m.IP)
isErrMount := m.IP == "bar" || m.IP == "baz"
if isErrMount && callCounts[m.IP] < 4 {
return errors.New("Fake error")
}
return nil
}
Convey("It should stop after the mount succeeds", func() {
So(r.restoreMounts(), ShouldBeNil)
So(callCounts, ShouldResemble, map[string]int{
"foo": 1,
"bar": 4, // 4th attempt should have succeeded
"baz": 4, // 4th attempt should have succeeded
})
})
// Skipping to avoid Wercker runtime making this test be flaky.
SkipConvey("With a failure pause", func() {
// Pause 10ms for each failure
r.restoreFailuresPause = 10 * time.Millisecond
Convey("It should pause between calls", func() {
start := time.Now()
So(r.restoreMounts(), ShouldBeNil)
runDur := time.Since(start)
// 10ms pauses for failures after the first one, 8 failures in total,
// 6 that should pause.
// Should almost take 60ms, plus some allowence for runtime.
So(runDur, ShouldAlmostEqual, 60*time.Millisecond, 10*time.Millisecond)
})
})
Convey("It should call in the expected order", func() {
So(r.restoreMounts(), ShouldBeNil)
So(callOrder, ShouldResemble, []string{
"foo",
"bar",
"baz",
"bar",
"baz",
"bar",
"baz",
"bar",
"baz",
})
})
})
})
}
func TestUnmountFolder(t *testing.T) {
Convey("Given a Remote with no mounts", t, func() {
kg := newMockKiteGetter()
r := Remote{
log: discardLogger,
mounts: mount.Mounts{},
kitesGetter: kg,
machines: machine.NewMachines(discardLogger, storage.NewMemoryStorage()),
localKite: &kite.Kite{
Id: "test id",
Config: &config.Config{
Username: "******",
},
},
}
Convey("When requesting a mount name and no path", func() {
Convey("It should return a no-mount-no-path error", func() {
var unmountPathCalled bool
r.unmountPath = func(p string) error {
unmountPathCalled = true
return nil
}
err := r.UnmountFolder(req.UnmountFolder{Name: "foo"})
So(err, ShouldNotBeNil)
So(unmountPathCalled, ShouldBeFalse)
kErr, ok := err.(*kite.Error)
So(ok, ShouldBeTrue)
So(kErr.Type, ShouldEqual, mountNotFoundNoPath)
})
})
})
Convey("Given a Remote with mounts", t, func() {
goodUnmountMocker := &unmountMocker{Name: "good mount"}
badUnmountMocker := &unmountMocker{
Name: "bad mount",
Error: errors.New("bad mount error"),
}
kg := newMockKiteGetter()
store := storage.NewMemoryStorage()
r := Remote{
log: testutil.DiscardLogger,
kitesGetter: kg,
storage: store,
machines: machine.NewMachines(discardLogger, store),
mounts: mount.Mounts{
&mount.Mount{
MountName: goodUnmountMocker.Name,
Unmounter: goodUnmountMocker,
},
&mount.Mount{
MountName: badUnmountMocker.Name,
Unmounter: badUnmountMocker,
},
},
localKite: &kite.Kite{
Id: "test id",
Config: &config.Config{
Username: "******",
},
},
}
Convey("When called with a name that exists", func() {
Convey("It should call the appropriate Unmounter", func() {
err := r.UnmountFolder(req.UnmountFolder{Name: goodUnmountMocker.Name})
So(err, ShouldBeNil)
So(goodUnmountMocker.Calls, ShouldEqual, 1)
So(badUnmountMocker.Calls, ShouldEqual, 0)
})
Convey("It should remove the mount from the Mounts slice", func() {
err := r.UnmountFolder(req.UnmountFolder{Name: goodUnmountMocker.Name})
So(err, ShouldBeNil)
So(len(r.mounts), ShouldEqual, 1)
// only the bad unmounter should be left
So(r.mounts[0].MountName, ShouldEqual, badUnmountMocker.Name)
})
})
Convey("When called with a mount that fails to unmount", func() {
Convey("It should still remove the mount from the Mounts slice", func() {
r.UnmountFolder(req.UnmountFolder{Name: badUnmountMocker.Name})
So(len(r.mounts), ShouldEqual, 1)
// only the good unmounter should be left
So(r.mounts[0].MountName, ShouldEqual, goodUnmountMocker.Name)
})
})
})
}