func (h *Helper) clusterClient(t *c.C) *cluster.Client {
if h.cluster == nil {
var err error
h.cluster, err = cluster.NewClientWithDial(nil, h.discoverdClient(t).NewServiceSet)
t.Assert(err, c.IsNil)
}
return h.cluster
}
func (s *SchedulerSuite) TestControllerRestart(t *c.C) {
// get the current controller details
app, err := s.controllerClient(t).GetApp("controller")
t.Assert(err, c.IsNil)
release, err := s.controllerClient(t).GetAppRelease("controller")
t.Assert(err, c.IsNil)
list, err := s.controllerClient(t).JobList("controller")
t.Assert(err, c.IsNil)
var jobs []*ct.Job
for _, job := range list {
if job.Type == "web" {
jobs = append(jobs, job)
}
}
t.Assert(jobs, c.HasLen, 1)
hostID, jobID, _ := cluster.ParseJobID(jobs[0].ID)
t.Assert(hostID, c.Not(c.Equals), "")
t.Assert(jobID, c.Not(c.Equals), "")
debugf(t, "current controller app[%s] host[%s] job[%s]", app.ID, hostID, jobID)
// start a second controller and wait for it to come up
stream, err := s.controllerClient(t).StreamJobEvents("controller", 0)
t.Assert(err, c.IsNil)
debug(t, "scaling the controller up")
t.Assert(s.controllerClient(t).PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: map[string]int{"web": 2, "scheduler": 1},
}), c.IsNil)
lastID, _ := waitForJobEvents(t, stream.Events, jobEvents{"web": {"up": 1}})
stream.Close()
// get direct client for new controller
var client *controller.Client
attempts := attempt.Strategy{
Total: 10 * time.Second,
Delay: 500 * time.Millisecond,
}
t.Assert(attempts.Run(func() (err error) {
set, err := s.discoverdClient(t).NewServiceSet("flynn-controller")
if err != nil {
return err
}
defer set.Close()
addrs := set.Addrs()
if len(addrs) != 2 {
return fmt.Errorf("expected 2 controller processes, got %d", len(addrs))
}
addr := addrs[1]
debug(t, "new controller address: ", addr)
client, err = controller.NewClient("http://"+addr, s.clusterConf(t).Key)
return
}), c.IsNil)
// kill the first controller and check the scheduler brings it back online
stream, err = client.StreamJobEvents("controller", lastID)
defer stream.Close()
t.Assert(err, c.IsNil)
cc, err := cluster.NewClientWithDial(nil, s.discoverdClient(t).NewServiceSet)
t.Assert(err, c.IsNil)
defer cc.Close()
hc, err := cc.DialHost(hostID)
t.Assert(err, c.IsNil)
defer hc.Close()
debug(t, "stopping job ", jobID)
t.Assert(hc.StopJob(jobID), c.IsNil)
waitForJobEvents(t, stream.Events, jobEvents{"web": {"down": 1, "up": 1}})
// scale back down
debug(t, "scaling the controller down")
t.Assert(s.controllerClient(t).PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: map[string]int{"web": 1, "scheduler": 1},
}), c.IsNil)
waitForJobEvents(t, stream.Events, jobEvents{"web": {"down": 1}})
// unset the suite's client so other tests use a new client
s.controller = nil
}