func (d *dummyNatsJobSupervisor) statusHandler(req boshhandler.Request) boshhandler.Response {
	switch req.Method {
	case "set_dummy_status":
		// Do not unmarshal message until determining its method
		var body map[string]string

		err := json.Unmarshal(req.GetPayload(), &body)
		if err != nil {
			return boshhandler.NewExceptionResponse(err)
		}

		d.status = body["status"]

		if d.status == "failing" && d.jobFailureHandler != nil {
			_ = d.jobFailureHandler(boshalert.MonitAlert{
				ID:          "fake-monit-alert",
				Service:     "fake-monit-service",
				Event:       "failing",
				Action:      "start",
				Date:        "Sun, 22 May 2011 20:07:41 +0500",
				Description: "fake-monit-description",
			})
		}

		return boshhandler.NewValueResponse("ok")
	default:
		return nil
	}
}
Esempio n. 2
0
func (dispatcher concreteActionDispatcher) dispatchAsynchronousAction(
	action boshaction.Action,
	req boshhandler.Request,
) boshhandler.Response {
	dispatcher.logger.Info(actionDispatcherLogTag, "Running async action %s", req.Method)

	var task boshtask.Task
	var err error

	runTask := func() (interface{}, error) {
		return dispatcher.actionRunner.Run(action, req.GetPayload())
	}

	cancelTask := func(_ boshtask.Task) error { return action.Cancel() }

	// Certain long-running tasks (e.g. configure_networks) must be resumed
	// after agent restart so that API consumers do not need to know
	// if agent is restarted midway through the task.
	if action.IsPersistent() {
		dispatcher.logger.Info(actionDispatcherLogTag, "Running persistent action %s", req.Method)
		task, err = dispatcher.taskService.CreateTask(runTask, cancelTask, dispatcher.removeInfo)
		if err != nil {
			err = bosherr.WrapErrorf(err, "Create Task Failed %s", req.Method)
			dispatcher.logger.Error(actionDispatcherLogTag, err.Error())
			return boshhandler.NewExceptionResponse(err)
		}

		taskInfo := boshtask.Info{
			TaskID:  task.ID,
			Method:  req.Method,
			Payload: req.GetPayload(),
		}

		err = dispatcher.taskManager.AddInfo(taskInfo)
		if err != nil {
			err = bosherr.WrapErrorf(err, "Action Failed %s", req.Method)
			dispatcher.logger.Error(actionDispatcherLogTag, err.Error())
			return boshhandler.NewExceptionResponse(err)
		}
	} else {
		task, err = dispatcher.taskService.CreateTask(runTask, cancelTask, nil)
		if err != nil {
			err = bosherr.WrapErrorf(err, "Create Task Failed %s", req.Method)
			dispatcher.logger.Error(actionDispatcherLogTag, err.Error())
			return boshhandler.NewExceptionResponse(err)
		}
	}

	dispatcher.taskService.StartTask(task)

	return boshhandler.NewValueResponse(boshtask.StateValue{
		AgentTaskID: task.ID,
		State:       task.State,
	})
}
Esempio n. 3
0
func (dispatcher concreteActionDispatcher) dispatchSynchronousAction(
	action boshaction.Action,
	req boshhandler.Request,
) boshhandler.Response {
	dispatcher.logger.Info(actionDispatcherLogTag, "Running sync action %s", req.Method)

	value, err := dispatcher.actionRunner.Run(action, req.GetPayload())
	if err != nil {
		err = bosherr.WrapErrorf(err, "Action Failed %s", req.Method)
		dispatcher.logger.Error(actionDispatcherLogTag, err.Error())
		return boshhandler.NewExceptionResponse(err)
	}

	return boshhandler.NewValueResponse(value)
}
		fs              *fakesys.FakeFileSystem
		receivedRequest boshhandler.Request
		httpClient      http.Client
	)

	BeforeEach(func() {
		serverURL = "https://*****:*****@127.0.0.1:6900"
		mbusURL, _ := url.Parse(serverURL)
		logger := boshlog.NewLogger(boshlog.LevelNone)
		fs = fakesys.NewFakeFileSystem()
		dirProvider := boshdir.NewProvider("/var/vcap")
		handler = NewHTTPSHandler(mbusURL, logger, fs, dirProvider)

		go handler.Start(func(req boshhandler.Request) (resp boshhandler.Response) {
			receivedRequest = req
			return boshhandler.NewValueResponse("expected value")
		})

		httpTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
		httpClient = http.Client{Transport: httpTransport}

		waitForServerToStart(serverURL, httpClient)
	})

	AfterEach(func() {
		handler.Stop()
		time.Sleep(1 * time.Millisecond)
	})

	Describe("POST /agent", func() {
		It("receives request and responds", func() {
Esempio n. 5
0
func init() {
	Describe("Agent", func() {
		var (
			logger           boshlog.Logger
			handler          *fakembus.FakeHandler
			platform         *fakeplatform.FakePlatform
			actionDispatcher *fakeagent.FakeActionDispatcher
			jobSupervisor    *fakejobsuper.FakeJobSupervisor
			specService      *fakeas.FakeV1Service
			syslogServer     *fakesyslog.FakeServer
			settingsService  *fakesettings.FakeSettingsService
			uuidGenerator    *fakeuuid.FakeGenerator
			timeService      *fakeclock.FakeClock
			agent            Agent
		)

		BeforeEach(func() {
			logger = boshlog.NewLogger(boshlog.LevelNone)
			handler = &fakembus.FakeHandler{}
			platform = fakeplatform.NewFakePlatform()
			actionDispatcher = &fakeagent.FakeActionDispatcher{}
			jobSupervisor = fakejobsuper.NewFakeJobSupervisor()
			specService = fakeas.NewFakeV1Service()
			syslogServer = &fakesyslog.FakeServer{}
			settingsService = &fakesettings.FakeSettingsService{}
			uuidGenerator = &fakeuuid.FakeGenerator{}
			timeService = fakeclock.NewFakeClock(time.Now())
			agent = New(
				logger,
				handler,
				platform,
				actionDispatcher,
				jobSupervisor,
				specService,
				syslogServer,
				5*time.Millisecond,
				settingsService,
				uuidGenerator,
				timeService,
			)
		})

		Describe("Run", func() {
			It("lets dispatcher handle requests arriving via handler", func() {
				err := agent.Run()
				Expect(err).ToNot(HaveOccurred())

				expectedResp := boshhandler.NewValueResponse("pong")
				actionDispatcher.DispatchResp = expectedResp

				req := boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
				resp := handler.RunFunc(req)

				Expect(actionDispatcher.DispatchReq).To(Equal(req))
				Expect(resp).To(Equal(expectedResp))
			})

			It("resumes persistent actions *before* dispatching new requests", func() {
				resumedBeforeStartingToDispatch := false
				handler.RunCallBack = func() {
					resumedBeforeStartingToDispatch = actionDispatcher.ResumedPreviouslyDispatchedTasks
				}

				err := agent.Run()
				Expect(err).ToNot(HaveOccurred())
				Expect(resumedBeforeStartingToDispatch).To(BeTrue())
			})

			Context("when heartbeats can be sent", func() {
				BeforeEach(func() {
					handler.KeepOnRunning()
				})

				BeforeEach(func() {
					jobName := "fake-job"
					nodeID := "node-id"
					jobIndex := 1
					specService.Spec = boshas.V1ApplySpec{
						JobSpec: boshas.JobSpec{Name: &jobName},
						Index:   &jobIndex,
						NodeID:  nodeID,
					}

					jobSupervisor.StatusStatus = "fake-state"

					platform.FakeVitalsService.GetVitals = boshvitals.Vitals{
						Load: []string{"a", "b", "c"},
					}
				})

				expectedJobName := "fake-job"
				expectedJobIndex := 1
				expectedNodeID := "node-id"
				expectedHb := Heartbeat{
					Job:      &expectedJobName,
					Index:    &expectedJobIndex,
					JobState: "fake-state",
					NodeID:   expectedNodeID,
					Vitals:   boshvitals.Vitals{Load: []string{"a", "b", "c"}},
				}

				It("sends initial heartbeat", func() {
					// Configure periodic heartbeat every 5 hours
					// so that we are sure that we will not receive it
					agent = New(
						logger,
						handler,
						platform,
						actionDispatcher,
						jobSupervisor,
						specService,
						syslogServer,
						5*time.Hour,
						settingsService,
						uuidGenerator,
						timeService,
					)

					// Immediately exit after sending initial heartbeat
					handler.SendErr = errors.New("stop")

					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("stop"))

					Expect(handler.SendInputs()).To(Equal([]fakembus.SendInput{
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
					}))
				})

				It("sends periodic heartbeats", func() {
					sentRequests := 0
					handler.SendCallback = func(_ fakembus.SendInput) {
						sentRequests++
						if sentRequests == 3 {
							handler.SendErr = errors.New("stop")
						}
					}

					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("stop"))

					Expect(handler.SendInputs()).To(Equal([]fakembus.SendInput{
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
					}))
				})
			})

			Context("when the agent fails to get job spec for a heartbeat", func() {
				BeforeEach(func() {
					specService.GetErr = errors.New("fake-spec-service-error")
					handler.KeepOnRunning()
				})

				It("returns the error", func() {
					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-spec-service-error"))
				})
			})

			Context("when the agent fails to get vitals for a heartbeat", func() {
				BeforeEach(func() {
					platform.FakeVitalsService.GetErr = errors.New("fake-vitals-service-error")
					handler.KeepOnRunning()
				})

				It("returns the error", func() {
					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-vitals-service-error"))
				})
			})

			It("sends job monitoring alerts to health manager", func() {
				handler.KeepOnRunning()

				monitAlert := boshalert.MonitAlert{
					ID:          "fake-monit-alert",
					Service:     "fake-service",
					Event:       "fake-event",
					Action:      "fake-action",
					Date:        "Sun, 22 May 2011 20:07:41 +0500",
					Description: "fake-description",
				}
				jobSupervisor.JobFailureAlert = &monitAlert

				// Fail the first time handler.Send is called for an alert (ignore heartbeats)
				handler.SendCallback = func(input fakembus.SendInput) {
					if input.Topic == boshhandler.Alert {
						handler.SendErr = errors.New("stop")
					}
				}

				err := agent.Run()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("stop"))

				expectedAlert := boshalert.Alert{
					ID:        "fake-monit-alert",
					Severity:  boshalert.SeverityDefault,
					Title:     "fake-service - fake-event - fake-action",
					Summary:   "fake-description",
					CreatedAt: int64(1306076861),
				}

				Expect(handler.SendInputs()).To(ContainElement(fakembus.SendInput{
					Target:  boshhandler.HealthMonitor,
					Topic:   boshhandler.Alert,
					Message: expectedAlert,
				}))
			})

			It("sends ssh alerts to health manager", func() {
				handler.KeepOnRunning()

				syslogMsg := boshsyslog.Msg{Content: "disconnected by user"}
				syslogServer.StartFirstSyslogMsg = &syslogMsg

				uuidGenerator.GeneratedUUID = "fake-uuid"

				// Fail the first time handler.Send is called for an alert (ignore heartbeats)
				handler.SendCallback = func(input fakembus.SendInput) {
					if input.Topic == boshhandler.Alert {
						handler.SendErr = errors.New("stop")
					}
				}

				err := agent.Run()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("stop"))

				expectedAlert := boshalert.Alert{
					ID:        "fake-uuid",
					Severity:  boshalert.SeverityWarning,
					Title:     "SSH Logout",
					Summary:   "disconnected by user",
					CreatedAt: timeService.Now().Unix(),
				}

				Expect(handler.SendInputs()).To(ContainElement(fakembus.SendInput{
					Target:  boshhandler.HealthMonitor,
					Topic:   boshhandler.Alert,
					Message: expectedAlert,
				}))
			})
		})
	})
}
func init() {
	Describe("natsHandler", func() {
		var (
			settingsService *fakesettings.FakeSettingsService
			client          *fakeyagnats.FakeYagnats
			logger          boshlog.Logger
			handler         boshhandler.Handler
		)

		BeforeEach(func() {
			settingsService = &fakesettings.FakeSettingsService{
				Settings: boshsettings.Settings{
					AgentID: "my-agent-id",
					Mbus:    "nats://*****:*****@127.0.0.1:1234",
				},
			}
			logger = boshlog.NewLogger(boshlog.LevelNone)
			client = fakeyagnats.New()
			handler = NewNatsHandler(settingsService, client, logger)
		})

		Describe("Start", func() {
			It("starts", func() {
				var receivedRequest boshhandler.Request

				handler.Start(func(req boshhandler.Request) (resp boshhandler.Response) {
					receivedRequest = req
					return boshhandler.NewValueResponse("expected value")
				})
				defer handler.Stop()

				Expect(client.ConnectedConnectionProvider()).ToNot(BeNil())

				Expect(client.SubscriptionCount()).To(Equal(1))
				subscriptions := client.Subscriptions("agent.my-agent-id")
				Expect(len(subscriptions)).To(Equal(1))

				expectedPayload := []byte(`{"method":"ping","arguments":["foo","bar"], "reply_to": "reply to me!"}`)
				subscription := subscriptions[0]
				subscription.Callback(&yagnats.Message{
					Subject: "agent.my-agent-id",
					Payload: expectedPayload,
				})

				Expect(receivedRequest).To(Equal(boshhandler.Request{
					ReplyTo: "reply to me!",
					Method:  "ping",
					Payload: expectedPayload,
				}))

				Expect(client.PublishedMessageCount()).To(Equal(1))
				messages := client.PublishedMessages("reply to me!")
				Expect(len(messages)).To(Equal(1))
				Expect(messages[0].Payload).To(Equal([]byte(`{"value":"expected value"}`)))
			})

			It("does not respond if the response is nil", func() {
				err := handler.Start(func(req boshhandler.Request) (resp boshhandler.Response) {
					return nil
				})
				Expect(err).ToNot(HaveOccurred())
				defer handler.Stop()

				subscription := client.Subscriptions("agent.my-agent-id")[0]
				subscription.Callback(&yagnats.Message{
					Subject: "agent.my-agent-id",
					Payload: []byte(`{"method":"ping","arguments":["foo","bar"], "reply_to": "reply to me!"}`),
				})

				Expect(client.PublishedMessageCount()).To(Equal(0))
			})

			It("responds with an error if the response is bigger than 1MB", func() {
				err := handler.Start(func(req boshhandler.Request) (resp boshhandler.Response) {
					// gets inflated by json.Marshal when enveloping
					size := 0

					switch req.Method {
					case "small":
						size = 1024*1024 - 12
					case "big":
						size = 1024 * 1024
					default:
						panic("unknown request size")
					}

					chars := make([]byte, size)
					for i := range chars {
						chars[i] = 'A'
					}
					return boshhandler.NewValueResponse(string(chars))
				})
				Expect(err).ToNot(HaveOccurred())
				defer handler.Stop()

				subscription := client.Subscriptions("agent.my-agent-id")[0]
				subscription.Callback(&yagnats.Message{
					Subject: "agent.my-agent-id",
					Payload: []byte(`{"method":"small","arguments":[], "reply_to": "fake-reply-to"}`),
				})

				subscription.Callback(&yagnats.Message{
					Subject: "agent.my-agent-id",
					Payload: []byte(`{"method":"big","arguments":[], "reply_to": "fake-reply-to"}`),
				})

				Expect(client.PublishedMessageCount()).To(Equal(1))
				messages := client.PublishedMessages("fake-reply-to")
				Expect(len(messages)).To(Equal(2))
				Expect(messages[0].Payload).To(MatchRegexp("value"))
				Expect(messages[1].Payload).To(Equal([]byte(
					`{"exception":{"message":"Response exceeded maximum allowed length"}}`)))
			})

			It("can add additional handler funcs to receive requests", func() {
				var firstHandlerReq, secondHandlerRequest boshhandler.Request

				handler.Start(func(req boshhandler.Request) (resp boshhandler.Response) {
					firstHandlerReq = req
					return boshhandler.NewValueResponse("first-handler-resp")
				})
				defer handler.Stop()

				handler.RegisterAdditionalFunc(func(req boshhandler.Request) (resp boshhandler.Response) {
					secondHandlerRequest = req
					return boshhandler.NewValueResponse("second-handler-resp")
				})

				expectedPayload := []byte(`{"method":"ping","arguments":["foo","bar"], "reply_to": "fake-reply-to"}`)

				subscription := client.Subscriptions("agent.my-agent-id")[0]
				subscription.Callback(&yagnats.Message{
					Subject: "agent.my-agent-id",
					Payload: expectedPayload,
				})

				// Expected requests received by both handlers
				Expect(firstHandlerReq).To(Equal(boshhandler.Request{
					ReplyTo: "fake-reply-to",
					Method:  "ping",
					Payload: expectedPayload,
				}))

				Expect(secondHandlerRequest).To(Equal(boshhandler.Request{
					ReplyTo: "fake-reply-to",
					Method:  "ping",
					Payload: expectedPayload,
				}))

				// Bosh handler responses were sent
				Expect(client.PublishedMessageCount()).To(Equal(1))
				messages := client.PublishedMessages("fake-reply-to")
				Expect(len(messages)).To(Equal(2))
				Expect(messages[0].Payload).To(Equal([]byte(`{"value":"first-handler-resp"}`)))
				Expect(messages[1].Payload).To(Equal([]byte(`{"value":"second-handler-resp"}`)))
			})

			It("has the correct connection info", func() {
				err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
				Expect(err).ToNot(HaveOccurred())
				defer handler.Stop()

				Expect(client.ConnectedConnectionProvider()).To(Equal(&yagnats.ConnectionInfo{
					Addr:     "127.0.0.1:1234",
					Username: "******",
					Password: "******",
				}))
			})

			It("does not err when no username and password", func() {
				settingsService.Settings.Mbus = "nats://127.0.0.1:1234"
				handler = NewNatsHandler(settingsService, client, logger)

				err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
				Expect(err).ToNot(HaveOccurred())
				defer handler.Stop()
			})

			It("errs when has username without password", func() {
				settingsService.Settings.Mbus = "nats://[email protected]:1234"
				handler = NewNatsHandler(settingsService, client, logger)

				err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
				Expect(err).To(HaveOccurred())
				defer handler.Stop()
			})
		})

		Describe("Send", func() {
			It("sends the message over nats to a subject that includes the target and topic", func() {
				errCh := make(chan error, 1)

				payload := map[string]string{"key1": "value1", "keyA": "valueA"}

				go func() {
					errCh <- handler.Send(boshhandler.HealthMonitor, boshhandler.Heartbeat, payload)
				}()

				var err error
				select {
				case err = <-errCh:
				}
				Expect(err).ToNot(HaveOccurred())

				Expect(client.PublishedMessageCount()).To(Equal(1))
				messages := client.PublishedMessages("hm.agent.heartbeat.my-agent-id")
				Expect(messages).To(HaveLen(1))
				Expect(messages[0].Payload).To(Equal(
					[]byte("{\"key1\":\"value1\",\"keyA\":\"valueA\"}"),
				))
			})
		})
	})
}
func init() {
	Describe("actionDispatcher", func() {
		var (
			logger        *fakes.FakeLogger
			taskService   *faketask.FakeService
			taskManager   *faketask.FakeManager
			actionFactory *fakeaction.FakeFactory
			actionRunner  *fakeaction.FakeRunner
			dispatcher    ActionDispatcher
		)

		BeforeEach(func() {
			logger = &fakes.FakeLogger{}
			taskService = faketask.NewFakeService()
			taskManager = faketask.NewFakeManager()
			actionFactory = fakeaction.NewFakeFactory()
			actionRunner = &fakeaction.FakeRunner{}
			dispatcher = NewActionDispatcher(logger, taskService, taskManager, actionFactory, actionRunner)
		})

		It("responds with exception when the method is unknown", func() {
			actionFactory.RegisterActionErr("fake-action", errors.New("fake-create-error"))

			req := boshhandler.NewRequest("fake-reply", "fake-action", []byte{})
			resp := dispatcher.Dispatch(req)
			boshassert.MatchesJSONString(GinkgoT(), resp, `{"exception":{"message":"unknown message fake-action"}}`)
		})

		Context("Action Payload Logging", func() {
			var (
				action *fakeaction.TestAction
				req    boshhandler.Request
			)

			Context("action is loggable", func() {
				BeforeEach(func() {
					req = boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
					action = &fakeaction.TestAction{Loggable: true}
					actionFactory.RegisterAction("fake-action", action)
					dispatcher.Dispatch(req)
				})

				It("logs the payload", func() {
					Expect(logger.DebugWithDetailsCallCount()).To(Equal(1))
					_, message, args := logger.DebugWithDetailsArgsForCall(0)
					Expect(message).To(Equal("Payload"))
					Expect(args[0]).To(Equal(req.Payload))
				})
			})

			Context("action is not loggable", func() {
				BeforeEach(func() {
					req = boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
					action = &fakeaction.TestAction{Loggable: false}
					actionFactory.RegisterAction("fake-action", action)
					dispatcher.Dispatch(req)
				})

				It("does not log the payload", func() {
					Expect(logger.DebugWithDetailsCallCount()).To(Equal(0))
				})
			})
		})

		Context("when action is synchronous", func() {
			var (
				req boshhandler.Request
			)

			BeforeEach(func() {
				req = boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
				actionFactory.RegisterAction("fake-action", &fakeaction.TestAction{Asynchronous: false})
			})

			It("handles synchronous action", func() {
				actionRunner.RunValue = "fake-value"

				resp := dispatcher.Dispatch(req)
				Expect(req.GetPayload()).To(Equal(actionRunner.RunPayload))
				Expect(boshhandler.NewValueResponse("fake-value")).To(Equal(resp))
			})

			It("handles synchronous action when err", func() {
				actionRunner.RunErr = errors.New("fake-run-error")

				resp := dispatcher.Dispatch(req)
				expectedJSON := fmt.Sprintf("{\"exception\":{\"message\":\"Action Failed %s: fake-run-error\"}}", req.Method)
				boshassert.MatchesJSONString(GinkgoT(), resp, expectedJSON)
			})
		})

		Context("when action is asynchronous", func() {
			var (
				req    boshhandler.Request
				action *fakeaction.TestAction
			)

			BeforeEach(func() {
				req = boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
				action = &fakeaction.TestAction{Asynchronous: true}
				actionFactory.RegisterAction("fake-action", action)
			})

			ItAllowsToCancelTask := func() {
				It("allows task to be cancelled", func() {
					dispatcher.Dispatch(req)

					err := taskService.StartedTasks["fake-generated-task-id"].Cancel()
					Expect(err).ToNot(HaveOccurred())

					Expect(action.Canceled).To(BeTrue())
				})

				It("returns error from cancelling task if canceling task fails", func() {
					action.CancelErr = errors.New("fake-cancel-err")
					dispatcher.Dispatch(req)

					err := taskService.StartedTasks["fake-generated-task-id"].Cancel()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-cancel-err"))
				})
			}

			Context("when action is not persistent", func() {
				BeforeEach(func() {
					action.Persistent = false
				})

				It("responds with task id and state", func() {
					resp := dispatcher.Dispatch(req)
					boshassert.MatchesJSONString(GinkgoT(), resp,
						`{"value":{"agent_task_id":"fake-generated-task-id","state":"running"}}`)
				})

				It("starts running created task", func() {
					dispatcher.Dispatch(req)
					Expect(len(taskService.StartedTasks)).To(Equal(1))
					Expect(taskService.StartedTasks["fake-generated-task-id"]).ToNot(BeNil())
				})

				It("returns create task error", func() {
					taskService.CreateTaskErr = errors.New("fake-create-task-error")
					resp := dispatcher.Dispatch(req)
					respJSON, err := json.Marshal(resp)
					Expect(err).ToNot(HaveOccurred())
					Expect(string(respJSON)).To(ContainSubstring("fake-create-task-error"))
				})

				It("return run value to the task", func() {
					actionRunner.RunValue = "fake-value"
					dispatcher.Dispatch(req)

					value, err := taskService.StartedTasks["fake-generated-task-id"].Func()
					Expect(value).To(Equal("fake-value"))
					Expect(err).ToNot(HaveOccurred())

					Expect(actionRunner.RunAction).To(Equal(action))
					Expect(string(actionRunner.RunPayload)).To(Equal("fake-payload"))
				})

				It("returns run error to the task", func() {
					actionRunner.RunErr = errors.New("fake-run-error")
					dispatcher.Dispatch(req)

					value, err := taskService.StartedTasks["fake-generated-task-id"].Func()
					Expect(value).To(BeNil())
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-run-error"))

					Expect(actionRunner.RunAction).To(Equal(action))
					Expect(string(actionRunner.RunPayload)).To(Equal("fake-payload"))
				})

				ItAllowsToCancelTask()

				It("does not add task to task manager since it should not be resumed if agent is restarted", func() {
					dispatcher.Dispatch(req)
					taskInfos, _ := taskManager.GetInfos()
					Expect(taskInfos).To(BeEmpty())
				})

				It("does not do anything after task finishes", func() {
					dispatcher.Dispatch(req)
					Expect(taskService.StartedTasks["fake-generated-task-id"].EndFunc).To(BeNil())
				})
			})

			Context("when action is persistent", func() {
				BeforeEach(func() {
					action.Persistent = true
				})

				It("responds with task id and state", func() {
					resp := dispatcher.Dispatch(req)
					boshassert.MatchesJSONString(GinkgoT(), resp,
						`{"value":{"agent_task_id":"fake-generated-task-id","state":"running"}}`)
				})

				It("starts running created task", func() {
					dispatcher.Dispatch(req)
					Expect(len(taskService.StartedTasks)).To(Equal(1))
					Expect(taskService.StartedTasks["fake-generated-task-id"]).ToNot(BeNil())
				})

				It("returns create task error", func() {
					taskService.CreateTaskErr = errors.New("fake-create-task-error")
					resp := dispatcher.Dispatch(req)
					respJSON, err := json.Marshal(resp)
					Expect(err).ToNot(HaveOccurred())
					Expect(string(respJSON)).To(ContainSubstring("fake-create-task-error"))
				})

				It("return run value to the task", func() {
					actionRunner.RunValue = "fake-value"
					dispatcher.Dispatch(req)

					value, err := taskService.StartedTasks["fake-generated-task-id"].Func()
					Expect(value).To(Equal("fake-value"))
					Expect(err).ToNot(HaveOccurred())

					Expect(actionRunner.RunAction).To(Equal(action))
					Expect(string(actionRunner.RunPayload)).To(Equal("fake-payload"))
				})

				It("returns run error to the task", func() {
					actionRunner.RunErr = errors.New("fake-run-error")
					dispatcher.Dispatch(req)

					value, err := taskService.StartedTasks["fake-generated-task-id"].Func()
					Expect(value).To(BeNil())
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-run-error"))

					Expect(actionRunner.RunAction).To(Equal(action))
					Expect(string(actionRunner.RunPayload)).To(Equal("fake-payload"))
				})

				ItAllowsToCancelTask()

				It("adds task to task manager before task starts so that it could be resumed if agent is restarted", func() {
					dispatcher.Dispatch(req)
					taskInfos, _ := taskManager.GetInfos()
					Expect(taskInfos).To(Equal([]boshtask.Info{
						boshtask.Info{
							TaskID:  "fake-generated-task-id",
							Method:  "fake-action",
							Payload: []byte("fake-payload"),
						},
					}))
				})

				It("removes task from task manager after task finishes", func() {
					dispatcher.Dispatch(req)
					taskService.StartedTasks["fake-generated-task-id"].EndFunc(boshtask.Task{ID: "fake-generated-task-id"})

					taskInfos, _ := taskManager.GetInfos()
					Expect(taskInfos).To(BeEmpty())
				})

				It("does not start running created task if task manager cannot add task", func() {
					taskManager.AddInfoErr = errors.New("fake-add-task-info-error")

					resp := dispatcher.Dispatch(req)
					boshassert.MatchesJSONString(GinkgoT(), resp,
						`{"exception":{"message":"Action Failed fake-action: fake-add-task-info-error"}}`)

					Expect(len(taskService.StartedTasks)).To(Equal(0))
				})
			})
		})

		Describe("ResumePreviouslyDispatchedTasks", func() {
			var firstAction, secondAction *fakeaction.TestAction

			BeforeEach(func() {
				err := taskManager.AddInfo(boshtask.Info{
					TaskID:  "fake-task-id-1",
					Method:  "fake-action-1",
					Payload: []byte("fake-task-payload-1"),
				})
				Expect(err).ToNot(HaveOccurred())

				err = taskManager.AddInfo(boshtask.Info{
					TaskID:  "fake-task-id-2",
					Method:  "fake-action-2",
					Payload: []byte("fake-task-payload-2"),
				})
				Expect(err).ToNot(HaveOccurred())

				firstAction = &fakeaction.TestAction{}
				secondAction = &fakeaction.TestAction{}
			})

			It("calls resume on each task that was saved in a task manager", func() {
				actionFactory.RegisterAction("fake-action-1", firstAction)
				actionFactory.RegisterAction("fake-action-2", secondAction)

				dispatcher.ResumePreviouslyDispatchedTasks()
				Expect(len(taskService.StartedTasks)).To(Equal(2))

				{ // Check that first task executes first action
					actionRunner.ResumeValue = "fake-resume-value-1"
					value, err := taskService.StartedTasks["fake-task-id-1"].Func()
					Expect(err).ToNot(HaveOccurred())
					Expect(value).To(Equal("fake-resume-value-1"))
					Expect(actionRunner.ResumeAction).To(Equal(firstAction))
					Expect(string(actionRunner.ResumePayload)).To(Equal("fake-task-payload-1"))
				}

				{ // Check that second task executes second action
					actionRunner.ResumeValue = "fake-resume-value-2"
					value, err := taskService.StartedTasks["fake-task-id-2"].Func()
					Expect(err).ToNot(HaveOccurred())
					Expect(value).To(Equal("fake-resume-value-2"))
					Expect(actionRunner.ResumeAction).To(Equal(secondAction))
					Expect(string(actionRunner.ResumePayload)).To(Equal("fake-task-payload-2"))
				}
			})

			It("removes tasks from task manager after each task finishes", func() {
				actionFactory.RegisterAction("fake-action-1", firstAction)
				actionFactory.RegisterAction("fake-action-2", secondAction)

				dispatcher.ResumePreviouslyDispatchedTasks()
				Expect(len(taskService.StartedTasks)).To(Equal(2))

				// Simulate all tasks ending
				taskService.StartedTasks["fake-task-id-1"].EndFunc(boshtask.Task{ID: "fake-task-id-1"})
				taskService.StartedTasks["fake-task-id-2"].EndFunc(boshtask.Task{ID: "fake-task-id-2"})

				taskInfos, err := taskManager.GetInfos()
				Expect(err).ToNot(HaveOccurred())
				Expect(taskInfos).To(BeEmpty())
			})

			It("return resume error to each task", func() {
				actionFactory.RegisterAction("fake-action-1", firstAction)
				actionFactory.RegisterAction("fake-action-2", secondAction)

				dispatcher.ResumePreviouslyDispatchedTasks()
				Expect(len(taskService.StartedTasks)).To(Equal(2))

				{ // Check that first task propagates its resume error
					actionRunner.ResumeErr = errors.New("fake-resume-error-1")
					value, err := taskService.StartedTasks["fake-task-id-1"].Func()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-resume-error-1"))
					Expect(value).To(BeNil())
					Expect(actionRunner.ResumeAction).To(Equal(firstAction))
					Expect(string(actionRunner.ResumePayload)).To(Equal("fake-task-payload-1"))
				}

				{ // Check that second task propagates its resume error
					actionRunner.ResumeErr = errors.New("fake-resume-error-2")
					value, err := taskService.StartedTasks["fake-task-id-2"].Func()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-resume-error-2"))
					Expect(value).To(BeNil())
					Expect(actionRunner.ResumeAction).To(Equal(secondAction))
					Expect(string(actionRunner.ResumePayload)).To(Equal("fake-task-payload-2"))
				}
			})

			It("ignores actions that cannot be created and removes them from task manager", func() {
				actionFactory.RegisterActionErr("fake-action-1", errors.New("fake-action-error-1"))
				actionFactory.RegisterAction("fake-action-2", secondAction)

				dispatcher.ResumePreviouslyDispatchedTasks()
				Expect(len(taskService.StartedTasks)).To(Equal(1))

				{ // Check that first action is removed from task manager
					taskInfos, err := taskManager.GetInfos()
					Expect(err).ToNot(HaveOccurred())
					Expect(taskInfos).To(Equal([]boshtask.Info{
						boshtask.Info{
							TaskID:  "fake-task-id-2",
							Method:  "fake-action-2",
							Payload: []byte("fake-task-payload-2"),
						},
					}))
				}

				{ // Check that second task executes second action
					taskService.StartedTasks["fake-task-id-2"].Func()
					Expect(actionRunner.ResumeAction).To(Equal(secondAction))
					Expect(string(actionRunner.ResumePayload)).To(Equal("fake-task-payload-2"))
				}
			})

			It("allows to cancel after resume", func() {
				actionFactory.RegisterAction("fake-action-1", firstAction)
				actionFactory.RegisterAction("fake-action-2", secondAction)

				dispatcher.ResumePreviouslyDispatchedTasks()

				err := taskService.StartedTasks["fake-task-id-1"].Cancel()
				Expect(err).ToNot(HaveOccurred())
				Expect(firstAction.Canceled).To(BeTrue())
				Expect(secondAction.Canceled).To(BeFalse())

				err = taskService.StartedTasks["fake-task-id-2"].Cancel()
				Expect(err).ToNot(HaveOccurred())
				Expect(secondAction.Canceled).To(BeTrue())
			})

			It("returns error from cancelling task when canceling resumed task fails", func() {
				actionFactory.RegisterAction("fake-action-1", firstAction)
				actionFactory.RegisterAction("fake-action-2", secondAction)

				dispatcher.ResumePreviouslyDispatchedTasks()

				firstAction.CancelErr = errors.New("fake-cancel-err-1")
				secondAction.CancelErr = errors.New("fake-cancel-err-2")

				err := taskService.StartedTasks["fake-task-id-1"].Cancel()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-cancel-err-1"))

				err = taskService.StartedTasks["fake-task-id-2"].Cancel()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-cancel-err-2"))
			})
		})
	})
}