func (s *S) TestJobLogWait(c *C) { app := s.createTestApp(c, &ct.App{Name: "joblog-wait"}) hostID, jobID := random.UUID(), random.UUID() hc := tu.NewFakeHostClient(hostID) hc.SetAttachFunc(jobID, func(req *host.AttachReq, wait bool) (cluster.AttachClient, error) { if !wait { return nil, cluster.ErrWouldWait } return cluster.NewAttachClient(newFakeLog(strings.NewReader("foo"))), nil }) s.cc.SetHostClient(hostID, hc) req, err := http.NewRequest("GET", fmt.Sprintf("%s/apps/%s/jobs/%s-%s/log", s.srv.URL, app.ID, hostID, jobID), nil) c.Assert(err, IsNil) req.SetBasicAuth("", authKey) res, err := http.DefaultClient.Do(req) c.Assert(err, IsNil) res.Body.Close() c.Assert(res.StatusCode, Equals, 404) req, err = http.NewRequest("GET", fmt.Sprintf("%s/apps/%s/jobs/%s-%s/log?wait=true", s.srv.URL, app.ID, hostID, jobID), nil) c.Assert(err, IsNil) req.SetBasicAuth("", authKey) res, err = http.DefaultClient.Do(req) var buf bytes.Buffer _, err = buf.ReadFrom(res.Body) res.Body.Close() c.Assert(err, IsNil) c.Assert(buf.String(), Equals, "foo") }
func (s *S) createLogTestApp(c *C, name string, stream io.Reader) (*ct.App, string, string) { app := s.createTestApp(c, &ct.App{Name: name}) hostID, jobID := random.UUID(), random.UUID() hc := tu.NewFakeHostClient(hostID) hc.SetAttach(jobID, cluster.NewAttachClient(newFakeLog(stream))) s.cc.SetHostClient(hostID, hc) return app, hostID, jobID }
func (s *S) TestKillJob(c *C) { app := s.createTestApp(c, &ct.App{Name: "killjob"}) hostID, jobID := random.UUID(), random.UUID() hc := tu.NewFakeHostClient(hostID) s.cc.AddHost(hc) c.Assert(s.c.DeleteJob(app.ID, hostID+"-"+jobID), IsNil) c.Assert(hc.IsStopped(jobID), Equals, true) }
func (s *S) TestKillJob(c *C) { app := s.createTestApp(c, &ct.App{Name: "killjob"}) hostID := fakeHostID() jobID := cluster.GenerateJobID(hostID) hc := tu.NewFakeHostClient(hostID) s.cc.AddHost(hc) c.Assert(s.c.DeleteJob(app.ID, jobID), IsNil) c.Assert(hc.IsStopped(jobID), Equals, true) }
func (s *S) TestKillJob(c *C) { app := s.createTestApp(c, &ct.App{Name: "killjob"}) hostID, jobID := random.UUID(), random.UUID() hc := tu.NewFakeHostClient(hostID) s.cc.SetHostClient(hostID, hc) res, err := s.Delete("/apps/" + app.ID + "/jobs/" + hostID + "-" + jobID) c.Assert(err, IsNil) c.Assert(res.StatusCode, Equals, 200) c.Assert(hc.IsStopped(jobID), Equals, true) }
func (s *S) TestRunJobDetached(c *C) { app := s.createTestApp(c, &ct.App{Name: "run-detached"}) artifact := s.createTestArtifact(c, &ct.Artifact{}) hostID := fakeHostID() host := tu.NewFakeHostClient(hostID, false) s.cc.AddHost(host) release := s.createTestRelease(c, &ct.Release{ ArtifactIDs: []string{artifact.ID}, Env: map[string]string{"RELEASE": "true", "FOO": "bar"}, }) args := []string{"foo", "bar"} req := &ct.NewJob{ ReleaseID: release.ID, ReleaseEnv: true, Args: args, Env: map[string]string{"JOB": "true", "FOO": "baz"}, Meta: map[string]string{"foo": "baz"}, } res, err := s.c.RunJobDetached(app.ID, req) c.Assert(err, IsNil) c.Assert(res.ID, Not(Equals), "") c.Assert(res.ReleaseID, Equals, release.ID) c.Assert(res.Type, Equals, "") c.Assert(res.Args, DeepEquals, args) jobs, err := host.ListJobs() c.Assert(err, IsNil) for _, j := range jobs { job := j.Job c.Assert(res.ID, Equals, job.ID) c.Assert(job.Metadata, DeepEquals, map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.app_name": app.Name, "flynn-controller.release": release.ID, "foo": "baz", }) c.Assert(job.Config.Args, DeepEquals, []string{"foo", "bar"}) c.Assert(job.Config.Env, DeepEquals, map[string]string{ "FLYNN_APP_ID": app.ID, "FLYNN_RELEASE_ID": release.ID, "FLYNN_PROCESS_TYPE": "", "FLYNN_JOB_ID": job.ID, "FOO": "baz", "JOB": "true", "RELEASE": "true", }) c.Assert(job.Config.Stdin, Equals, false) } }
func (s *S) TestJobRestartBackoffPolicy(c *C) { // Create a fake cluster with an existing running formation appID := "app" artifact := &ct.Artifact{ID: "artifact", Type: "docker", URI: "docker://foo/bar"} processes := map[string]int{"web": 2} release := newRelease("release", artifact, processes) cc := newFakeControllerClient(appID, release, artifact, processes, nil) hostID := "host0" cl := newFakeCluster(hostID, appID, release.ID, processes, nil) hc := tu.NewFakeHostClient(hostID) cl.SetHostClient(hostID, hc) cx := newContext(cc, cl) events := make(chan *host.Event, 2) defer close(events) cx.syncCluster(events) c.Assert(cx.jobs.Len(), Equals, 2) // Give the watchHost goroutine chance to start waitForWatchHostStart(events, c) durations := make([]time.Duration, 0) timeAfterFunc = testAfterFunc(&durations) // First restart: scheduled immediately cl.RemoveJob(hostID, "job0", false) e := waitForJobStartEvent(events, c) c.Assert(len(durations), Equals, 0) // Second restart: scheduled for 1 * backoffPeriod cl.RemoveJob(hostID, e.JobID, false) e = waitForJobStartEvent(events, c) c.Assert(len(durations), Equals, 1) c.Assert(durations[0], Equals, backoffPeriod) // Third restart: scheduled for 2 * backoffPeriod cl.RemoveJob(hostID, e.JobID, false) e = waitForJobStartEvent(events, c) c.Assert(len(durations), Equals, 2) c.Assert(durations[1], Equals, 2*backoffPeriod) // After backoffPeriod has elapsed: scheduled immediately job := cx.jobs.Get(hostID, e.JobID) job.startedAt = time.Now().Add(-backoffPeriod - 1*time.Second) cl.RemoveJob(hostID, job.ID, false) waitForJobStartEvent(events, c) c.Assert(len(durations), Equals, 2) }
func (s *S) TestRunJobDetached(c *C) { app := s.createTestApp(c, &ct.App{Name: "run-detached"}) hostID := random.UUID() host := tu.NewFakeHostClient(hostID) s.cc.AddHost(host) artifact := s.createTestArtifact(c, &ct.Artifact{Type: "docker", URI: "docker://foo/bar"}) release := s.createTestRelease(c, &ct.Release{ ArtifactID: artifact.ID, Env: map[string]string{"RELEASE": "true", "FOO": "bar"}, }) cmd := []string{"foo", "bar"} req := &ct.NewJob{ ReleaseID: release.ID, ReleaseEnv: true, Cmd: cmd, Env: map[string]string{"JOB": "true", "FOO": "baz"}, Meta: map[string]string{"foo": "baz"}, } res, err := s.c.RunJobDetached(app.ID, req) c.Assert(err, IsNil) c.Assert(res.ID, Not(Equals), "") c.Assert(res.ReleaseID, Equals, release.ID) c.Assert(res.Type, Equals, "") c.Assert(res.Cmd, DeepEquals, cmd) job := host.Jobs[0] c.Assert(res.ID, Equals, hostID+"-"+job.ID) c.Assert(job.Metadata, DeepEquals, map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.app_name": app.Name, "flynn-controller.release": release.ID, "foo": "baz", }) c.Assert(job.Config.Cmd, DeepEquals, []string{"foo", "bar"}) c.Assert(job.Config.Env, DeepEquals, map[string]string{ "FLYNN_APP_ID": app.ID, "FLYNN_RELEASE_ID": release.ID, "FLYNN_PROCESS_TYPE": "", "FLYNN_JOB_ID": hostID + "-" + job.ID, "FOO": "baz", "JOB": "true", "RELEASE": "true", }) c.Assert(job.Config.Stdin, Equals, false) }
func newFakeCluster(hostID, appID, releaseID string, processes map[string]int, jobs []*host.Job) *tu.FakeCluster { if jobs == nil { jobs = make([]*host.Job, 0) } for t, c := range processes { for i := 0; i < c; i++ { job := &host.Job{ ID: fmt.Sprintf("job%d", i), Metadata: map[string]string{ "flynn-controller.app": appID, "flynn-controller.release": releaseID, "flynn-controller.type": t, }, } jobs = append(jobs, job) } } cl := tu.NewFakeCluster() cl.SetHosts(map[string]host.Host{hostID: {ID: hostID, Jobs: jobs}}) cl.SetHostClient(hostID, tu.NewFakeHostClient(hostID)) return cl }
func (s *S) TestKillJob(c *C) { app := s.createTestApp(c, &ct.App{Name: "killjob"}) release := s.createTestRelease(c, &ct.Release{}) hostID := fakeHostID() uuid := random.UUID() jobID := cluster.GenerateJobID(hostID, uuid) s.createTestJob(c, &ct.Job{ ID: jobID, UUID: uuid, HostID: hostID, AppID: app.ID, ReleaseID: release.ID, Type: "web", State: ct.JobStateStarting, Meta: map[string]string{"some": "info"}, }) hc := tu.NewFakeHostClient(hostID, false) hc.AddJob(&host.Job{ID: jobID}) s.cc.AddHost(hc) err := s.c.DeleteJob(app.ID, jobID) c.Assert(err, IsNil) c.Assert(hc.IsStopped(jobID), Equals, true) }
func (s *S) TestRunJobAttached(c *C) { app := s.createTestApp(c, &ct.App{Name: "run-attached"}) hostID := fakeHostID() hc := tu.NewFakeHostClient(hostID, false) s.cc.AddHost(hc) input := make(chan string, 1) var jobID string hc.SetAttachFunc("*", func(req *host.AttachReq, wait bool) (cluster.AttachClient, error) { c.Assert(wait, Equals, true) c.Assert(req.JobID, Not(Equals), "") c.Assert(req, DeepEquals, &host.AttachReq{ JobID: req.JobID, Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagStdin | host.AttachFlagStream, Height: 20, Width: 10, }) jobID = req.JobID inPipeR, inPipeW := io.Pipe() go func() { buf := make([]byte, 10) n, _ := inPipeR.Read(buf) input <- string(buf[:n]) }() outPipeR, outPipeW := io.Pipe() go outPipeW.Write([]byte("test out")) return cluster.NewAttachClient(struct { io.Reader io.WriteCloser }{outPipeR, inPipeW}), nil }) artifact := s.createTestArtifact(c, &ct.Artifact{}) release := s.createTestRelease(c, &ct.Release{ ArtifactIDs: []string{artifact.ID}, Env: map[string]string{"RELEASE": "true", "FOO": "bar"}, }) data := &ct.NewJob{ ReleaseID: release.ID, ReleaseEnv: true, Args: []string{"foo", "bar"}, Env: map[string]string{"JOB": "true", "FOO": "baz"}, Meta: map[string]string{"foo": "baz"}, TTY: true, Columns: 10, Lines: 20, } rwc, err := s.c.RunJobAttached(app.ID, data) c.Assert(err, IsNil) _, err = rwc.Write([]byte("test in")) c.Assert(err, IsNil) c.Assert(<-input, Equals, "test in") buf := make([]byte, 10) n, _ := rwc.Read(buf) c.Assert(err, IsNil) c.Assert(string(buf[:n]), Equals, "test out") rwc.Close() jobs, err := hc.ListJobs() c.Assert(err, IsNil) for _, j := range jobs { job := j.Job c.Assert(job.ID, Equals, jobID) c.Assert(job.Metadata, DeepEquals, map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.app_name": app.Name, "flynn-controller.release": release.ID, "foo": "baz", }) c.Assert(job.Config.Args, DeepEquals, []string{"foo", "bar"}) c.Assert(job.Config.Env, DeepEquals, map[string]string{ "FLYNN_APP_ID": app.ID, "FLYNN_RELEASE_ID": release.ID, "FLYNN_PROCESS_TYPE": "", "FLYNN_JOB_ID": job.ID, "FOO": "baz", "JOB": "true", "RELEASE": "true", }) c.Assert(job.Config.Stdin, Equals, true) } }
func (s *S) TestRunJobAttached(c *C) { app := s.createTestApp(c, &ct.App{Name: "run-attached"}) hostID := random.UUID() hc := tu.NewFakeHostClient(hostID) done := make(chan struct{}) var jobID string hc.SetAttachFunc("*", func(req *host.AttachReq, wait bool) (cluster.AttachClient, error) { c.Assert(wait, Equals, true) c.Assert(req.JobID, Not(Equals), "") c.Assert(req, DeepEquals, &host.AttachReq{ JobID: req.JobID, Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagStdin | host.AttachFlagStream, Height: 20, Width: 10, }) jobID = req.JobID pipeR, pipeW := io.Pipe() go func() { stdin, err := ioutil.ReadAll(pipeR) c.Assert(err, IsNil) c.Assert(string(stdin), Equals, "test in") close(done) }() return cluster.NewAttachClient(struct { io.Reader io.WriteCloser }{strings.NewReader("test out"), pipeW}), nil }) s.cc.SetHostClient(hostID, hc) s.cc.SetHosts(map[string]host.Host{hostID: {}}) artifact := s.createTestArtifact(c, &ct.Artifact{Type: "docker", URI: "docker://foo/bar"}) release := s.createTestRelease(c, &ct.Release{ ArtifactID: artifact.ID, Env: map[string]string{"RELEASE": "true", "FOO": "bar"}, }) data, _ := json.Marshal(&ct.NewJob{ ReleaseID: release.ID, Cmd: []string{"foo", "bar"}, Env: map[string]string{"JOB": "true", "FOO": "baz"}, TTY: true, Columns: 10, Lines: 20, }) req, err := http.NewRequest("POST", s.srv.URL+"/apps/"+app.ID+"/jobs", bytes.NewBuffer(data)) c.Assert(err, IsNil) req.SetBasicAuth("", authKey) req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/vnd.flynn.attach") _, rwc, err := utils.HijackRequest(req, nil) c.Assert(err, IsNil) _, err = rwc.Write([]byte("test in")) c.Assert(err, IsNil) rwc.CloseWrite() stdout, err := ioutil.ReadAll(rwc) c.Assert(err, IsNil) c.Assert(string(stdout), Equals, "test out") rwc.Close() job := s.cc.GetHost(hostID).Jobs[0] c.Assert(job.ID, Equals, jobID) c.Assert(job.Metadata, DeepEquals, map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.release": release.ID, }) c.Assert(job.Config.Cmd, DeepEquals, []string{"foo", "bar"}) c.Assert(job.Config.Env, DeepEquals, map[string]string{"FOO": "baz", "JOB": "true", "RELEASE": "true"}) c.Assert(job.Config.Stdin, Equals, true) }
func (s *S) TestWatchHost(c *C) { // Create a fake cluster with an existing running formation and a one-off job appID := "app" artifact := &ct.Artifact{ID: "artifact", Type: "docker", URI: "docker://foo/bar"} processes := map[string]int{"web": 3} release := newRelease("release", artifact, processes) cc := newFakeControllerClient(appID, release, artifact, processes, nil) hostID := "host0" cl := newFakeCluster(hostID, appID, release.ID, processes, []*host.Job{ {ID: "one-off-job", Metadata: map[string]string{"flynn-controller.app": appID, "flynn-controller.release": release.ID}}, }) hc := tu.NewFakeHostClient(hostID) cl.SetHostClient(hostID, hc) cx := newContext(cc, cl) events := make(chan *host.Event, 4) defer close(events) cx.syncCluster(events) c.Assert(cx.jobs.Len(), Equals, 4) c.Assert(len(cl.GetHost(hostID).Jobs), Equals, 4) // Give the watchHost goroutine chance to start waitForWatchHostStart(events, c) // Check jobs are marked as up once started hc.SendEvent("start", "job0") hc.SendEvent("start", "job1") hc.SendEvent("start", "job2") hc.SendEvent("start", "one-off-job") waitForHostEvents(4, events, c) c.Assert(len(cc.jobs), Equals, 4) c.Assert(cc.jobs[hostID+"-job0"].State, Equals, "up") c.Assert(cc.jobs[hostID+"-job1"].State, Equals, "up") c.Assert(cc.jobs[hostID+"-job2"].State, Equals, "up") c.Assert(cc.jobs[hostID+"-one-off-job"].State, Equals, "up") // Check that when a formation's job is removed, it is marked as down and a new one is scheduled cl.RemoveJob(hostID, "job0", false) waitForHostEvents(2, events, c) // wait for both a stop and start event c.Assert(cc.jobs[hostID+"-job0"].State, Equals, "down") c.Assert(cx.jobs.Len(), Equals, 4) c.Assert(len(cl.GetHost(hostID).Jobs), Equals, 4) job, _ := hc.GetJob("job0") c.Assert(job, IsNil) // Check that when a one-off job is removed, it is marked as down but a new one is not scheduled cl.RemoveJob(hostID, "one-off-job", false) waitForHostEvents(1, events, c) c.Assert(cc.jobs[hostID+"-one-off-job"].State, Equals, "down") c.Assert(cx.jobs.Len(), Equals, 3) c.Assert(len(cl.GetHost(hostID).Jobs), Equals, 3) job, _ = hc.GetJob("one-off-job") c.Assert(job, IsNil) // Check that when a job errors, it is marked as crashed and a new one is started cl.RemoveJob(hostID, "job1", true) waitForHostEvents(2, events, c) // wait for both an error and start event c.Assert(cc.jobs[hostID+"-job1"].State, Equals, "crashed") c.Assert(cx.jobs.Len(), Equals, 3) c.Assert(len(cl.GetHost(hostID).Jobs), Equals, 3) job, _ = hc.GetJob("job1") c.Assert(job, IsNil) // Check that a new host gets detected host2ID := "host2" cl.AddHost(host2ID, host.Host{ID: host2ID}) cl.SetHostClient(host2ID, tu.NewFakeHostClient(host2ID)) cl.SendEvent(host2ID, "add") // wait for host to get up waitForWatchHostStart(events, c) host2 := cl.GetHost(host2ID) c.Assert(len(host2.Jobs), Equals, 0) }
func (s *S) TestOmni(c *C) { // Create a fake cluster with an existing running formation appID := "existing-app" artifact := &ct.Artifact{ID: "existing-artifact"} processes := make(map[string]int) release := newRelease("existing-release", artifact, processes) stream := make(chan *ct.ExpandedFormation) cc := newFakeControllerClient(appID, release, artifact, processes, stream) hostID := "host0" cl := newFakeCluster(hostID, appID, release.ID, processes, nil) // inject another host host1ID := "host1" cl.AddHost(host1ID, host.Host{ID: host1ID}) cl.SetHostClient(host1ID, tu.NewFakeHostClient(host1ID)) cx := newContext(cc, cl) events := make(chan *FormationEvent) defer close(events) hostEvents := make(chan *host.Event, 14) defer close(hostEvents) go cx.watchFormations(events, hostEvents) // Give the scheduler chance to sync with the cluster, then check it's in sync waitForFormationEvent(events, c) waitForWatchHostStart(hostEvents, c) waitForWatchHostStart(hostEvents, c) f := &ct.ExpandedFormation{ App: &ct.App{ID: "app0"}, Release: &ct.Release{ ID: "release0", ArtifactID: "artifact0", Processes: map[string]ct.ProcessType{ "web": {Cmd: []string{"start", "web"}, Omni: true}, "worker": {Cmd: []string{"start", "worker"}}, }, }, Artifact: &ct.Artifact{ID: "artifact0", Type: "docker", URI: "docker://foo/bar"}, UpdatedAt: time.Now(), } updates := []*formationUpdate{ {processes: map[string]int{"web": 2}}, {processes: map[string]int{"web": 3, "worker": 1}}, {processes: map[string]int{"web": 1}}, } for _, u := range updates { f.Processes = u.processes stream <- f waitForFormationEvent(events, c) host := cl.GetHost(hostID) host1 := cl.GetHost(host1ID) c.Assert(len(host.Jobs)+len(host1.Jobs), Equals, u.processes["web"]*2+u.processes["worker"]) } waitForHostEvents(12, hostEvents, c) // Check that a new host gets omni jobs host2ID := "host2" cl.AddHost(host2ID, host.Host{ID: host2ID}) cl.SetHostClient(host2ID, tu.NewFakeHostClient(host2ID)) cl.SendEvent(host2ID, "add") // wait for host to get up waitForWatchHostStart(hostEvents, c) waitForHostEvents(1, hostEvents, c) // wait for the job to get scheduled host2 := cl.GetHost(host2ID) c.Assert(len(host2.Jobs), Equals, 1) }
func (s *S) TestRunJobAttached(c *C) { app := s.createTestApp(c, &ct.App{Name: "run-attached"}) hostID := fakeHostID() hc := tu.NewFakeHostClient(hostID) s.cc.AddHost(hc) done := make(chan struct{}) var jobID string hc.SetAttachFunc("*", func(req *host.AttachReq, wait bool) (cluster.AttachClient, error) { c.Assert(wait, Equals, true) c.Assert(req.JobID, Not(Equals), "") c.Assert(req, DeepEquals, &host.AttachReq{ JobID: req.JobID, Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagStdin | host.AttachFlagStream, Height: 20, Width: 10, }) jobID = req.JobID pipeR, pipeW := io.Pipe() go func() { stdin, err := ioutil.ReadAll(pipeR) c.Assert(err, IsNil) c.Assert(string(stdin), Equals, "test in") close(done) }() return cluster.NewAttachClient(struct { io.Reader io.WriteCloser }{strings.NewReader("test out"), pipeW}), nil }) artifact := s.createTestArtifact(c, &ct.Artifact{Type: "docker", URI: "docker://foo/bar"}) release := s.createTestRelease(c, &ct.Release{ ArtifactID: artifact.ID, Env: map[string]string{"RELEASE": "true", "FOO": "bar"}, }) data := &ct.NewJob{ ReleaseID: release.ID, ReleaseEnv: true, Cmd: []string{"foo", "bar"}, Env: map[string]string{"JOB": "true", "FOO": "baz"}, Meta: map[string]string{"foo": "baz"}, TTY: true, Columns: 10, Lines: 20, } rwc, err := s.c.RunJobAttached(app.ID, data) c.Assert(err, IsNil) _, err = rwc.Write([]byte("test in")) c.Assert(err, IsNil) rwc.CloseWrite() stdout, err := ioutil.ReadAll(rwc) c.Assert(err, IsNil) c.Assert(string(stdout), Equals, "test out") rwc.Close() job := hc.Jobs[0] c.Assert(job.ID, Equals, jobID) c.Assert(job.Metadata, DeepEquals, map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.app_name": app.Name, "flynn-controller.release": release.ID, "foo": "baz", }) c.Assert(job.Config.Cmd, DeepEquals, []string{"foo", "bar"}) c.Assert(job.Config.Env, DeepEquals, map[string]string{ "FLYNN_APP_ID": app.ID, "FLYNN_RELEASE_ID": release.ID, "FLYNN_PROCESS_TYPE": "", "FLYNN_JOB_ID": job.ID, "FOO": "baz", "JOB": "true", "RELEASE": "true", }) c.Assert(job.Config.Stdin, Equals, true) }