func (h *Helper) clusterClient(t *c.C) *cluster.Client {
h.clusterMtx.Lock()
defer h.clusterMtx.Unlock()
if h.cluster == nil {
h.cluster = cluster.NewClientWithServices(h.discoverdClient(t).Service)
}
return h.cluster
}
func (s *SchedulerSuite) TestControllerRestart(t *c.C) {
// get the current controller details
app, err := s.controllerClient(t).GetApp("controller")
t.Assert(err, c.IsNil)
release, err := s.controllerClient(t).GetAppRelease("controller")
t.Assert(err, c.IsNil)
formation, err := s.controllerClient(t).GetFormation(app.ID, release.ID)
t.Assert(err, c.IsNil)
list, err := s.controllerClient(t).JobList("controller")
t.Assert(err, c.IsNil)
var jobs []*ct.Job
for _, job := range list {
if job.Type == "web" && job.State == "up" {
jobs = append(jobs, job)
}
}
t.Assert(jobs, c.HasLen, formation.Processes["web"])
jobID := jobs[0].ID
hostID, _ := cluster.ExtractHostID(jobID)
t.Assert(hostID, c.Not(c.Equals), "")
debugf(t, "current controller app[%s] host[%s] job[%s]", app.ID, hostID, jobID)
// start another controller and wait for it to come up
watcher, err := s.controllerClient(t).WatchJobEvents("controller", release.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
debug(t, "scaling the controller up")
formation.Processes["web"]++
t.Assert(s.controllerClient(t).PutFormation(formation), c.IsNil)
err = watcher.WaitFor(ct.JobEvents{"web": {"up": 1}}, scaleTimeout, nil)
t.Assert(err, c.IsNil)
// kill the first controller and check the scheduler brings it back online
cc := cluster.NewClientWithServices(s.discoverdClient(t).Service)
hc, err := cc.Host(hostID)
t.Assert(err, c.IsNil)
debug(t, "stopping job ", jobID)
t.Assert(hc.StopJob(jobID), c.IsNil)
err = watcher.WaitFor(ct.JobEvents{"web": {"down": 1, "up": 1}}, scaleTimeout, nil)
t.Assert(err, c.IsNil)
// scale back down
debug(t, "scaling the controller down")
formation.Processes["web"]--
t.Assert(s.controllerClient(t).PutFormation(formation), c.IsNil)
err = watcher.WaitFor(ct.JobEvents{"web": {"down": 1}}, scaleTimeout, nil)
t.Assert(err, c.IsNil)
// unset the suite's client so other tests use a new client
s.controller = nil
}
func (api *HTTPAPI) ConfigureClusterClient(discoverdURL string) {
disc := discoverd.NewClientWithURL(discoverdURL)
api.cluster.Store(cluster.NewClientWithServices(disc.Service))
}
func (s *SchedulerSuite) TestControllerRestart(t *c.C) {
// get the current controller details
app, err := s.controllerClient(t).GetApp("controller")
t.Assert(err, c.IsNil)
release, err := s.controllerClient(t).GetAppRelease("controller")
t.Assert(err, c.IsNil)
formation, err := s.controllerClient(t).GetFormation(app.ID, release.ID)
t.Assert(err, c.IsNil)
list, err := s.controllerClient(t).JobList("controller")
t.Assert(err, c.IsNil)
var jobs []*ct.Job
for _, job := range list {
if job.Type == "web" && job.State == ct.JobStateUp {
jobs = append(jobs, job)
}
}
t.Assert(jobs, c.HasLen, formation.Processes["web"])
jobID := jobs[0].ID
hostID, _ := cluster.ExtractHostID(jobID)
t.Assert(hostID, c.Not(c.Equals), "")
debugf(t, "current controller app[%s] host[%s] job[%s]", app.ID, hostID, jobID)
// subscribe to service events, wait for current event
events := make(chan *discoverd.Event)
stream, err := s.discoverdClient(t).Service("controller").Watch(events)
t.Assert(err, c.IsNil)
defer stream.Close()
type serviceEvents map[discoverd.EventKind]int
wait := func(expected serviceEvents) {
actual := make(serviceEvents)
outer:
for {
select {
case event := <-events:
actual[event.Kind]++
for kind, count := range expected {
if actual[kind] != count {
continue outer
}
}
return
case <-time.After(scaleTimeout):
t.Fatal("timed out waiting for controller service event")
}
}
}
wait(serviceEvents{discoverd.EventKindCurrent: 1})
// start another controller and wait for it to come up
debug(t, "scaling the controller up")
formation.Processes["web"]++
t.Assert(s.controllerClient(t).PutFormation(formation), c.IsNil)
wait(serviceEvents{discoverd.EventKindUp: 1})
// kill the first controller and check the scheduler brings it back online
cc := cluster.NewClientWithServices(s.discoverdClient(t).Service)
hc, err := cc.Host(hostID)
t.Assert(err, c.IsNil)
debug(t, "stopping job ", jobID)
t.Assert(hc.StopJob(jobID), c.IsNil)
wait(serviceEvents{discoverd.EventKindUp: 1, discoverd.EventKindDown: 1})
// scale back down
debug(t, "scaling the controller down")
formation.Processes["web"]--
t.Assert(s.controllerClient(t).PutFormation(formation), c.IsNil)
wait(serviceEvents{discoverd.EventKindDown: 1})
// unset the suite's client so other tests use a new client
s.controller = nil
}
func (s *SchedulerSuite) TestControllerRestart(t *c.C) {
// get the current controller details
app, err := s.controllerClient(t).GetApp("controller")
t.Assert(err, c.IsNil)
release, err := s.controllerClient(t).GetAppRelease("controller")
t.Assert(err, c.IsNil)
formation, err := s.controllerClient(t).GetFormation(app.ID, release.ID)
t.Assert(err, c.IsNil)
list, err := s.controllerClient(t).JobList("controller")
t.Assert(err, c.IsNil)
var jobs []*ct.Job
for _, job := range list {
if job.Type == "web" && job.State == "up" {
jobs = append(jobs, job)
}
}
t.Assert(jobs, c.HasLen, 2)
hostID, jobID, _ := cluster.ParseJobID(jobs[0].ID)
t.Assert(hostID, c.Not(c.Equals), "")
t.Assert(jobID, c.Not(c.Equals), "")
debugf(t, "current controller app[%s] host[%s] job[%s]", app.ID, hostID, jobID)
// start a second controller and wait for it to come up
events := make(chan *ct.JobEvent)
stream, err := s.controllerClient(t).StreamJobEvents("controller", 0, events)
t.Assert(err, c.IsNil)
debug(t, "scaling the controller up")
formation.Processes["web"]++
t.Assert(s.controllerClient(t).PutFormation(formation), c.IsNil)
lastID, _ := waitForJobEvents(t, stream, events, jobEvents{"web": {"up": 1}})
stream.Close()
// get direct client for new controller
var client *controller.Client
attempts := attempt.Strategy{
Total: 10 * time.Second,
Delay: 500 * time.Millisecond,
}
t.Assert(attempts.Run(func() (err error) {
addrs, err := s.discoverdClient(t).Service("flynn-controller").Addrs()
if err != nil {
return err
}
if len(addrs) != 3 {
return fmt.Errorf("expected 3 controller processes, got %d", len(addrs))
}
addr := addrs[2]
debug(t, "new controller address: ", addr)
client, err = controller.NewClient("http://"+addr, s.clusterConf(t).Key)
if err != nil {
return err
}
events = make(chan *ct.JobEvent)
stream, err = client.StreamJobEvents("controller", lastID, events)
return
}), c.IsNil)
defer stream.Close()
// kill the first controller and check the scheduler brings it back online
cc, err := cluster.NewClientWithServices(s.discoverdClient(t).Service)
t.Assert(err, c.IsNil)
hc, err := cc.DialHost(hostID)
t.Assert(err, c.IsNil)
debug(t, "stopping job ", jobID)
t.Assert(hc.StopJob(jobID), c.IsNil)
waitForJobEvents(t, stream, events, jobEvents{"web": {"down": 1, "up": 1}})
// scale back down
debug(t, "scaling the controller down")
formation.Processes["web"]--
t.Assert(s.controllerClient(t).PutFormation(formation), c.IsNil)
waitForJobEvents(t, stream, events, jobEvents{"web": {"down": 1}})
// unset the suite's client so other tests use a new client
s.controller = nil
}