func (kicker *HurlerKicker) Complete(task *models.Task) { log.Println("kicking complete") json := task.ToJSON() res, err := http.DefaultClient.Do(&http.Request{ Method: "POST", URL: &url.URL{ Scheme: "http", Host: kicker.hurlerAddress, Path: "/tasks", }, Host: "stager", Body: ioutil.NopCloser(bytes.NewBuffer(json)), ContentLength: int64(len(json)), Header: map[string][]string{ "Content-Type": []string{"application/json"}, }, }) if err != nil { log.Println("kick complete error", err) return } log.Println("kick complete", res.Status) }
func (handler *Handler) resolveTask(task *models.Task) { err := handler.bbs.ResolvingTask(task) if err != nil { logger.Info("handler.resolving-failed", map[string]interface{}{ "task": task.Guid, "error": err.Error(), }) return } err = handler.natsClient.Publish(task.ReplyTo, task.ToJSON()) if err != nil { logger.Error("handler.publish-failed", map[string]interface{}{ "task": task.Guid, "error": err.Error(), }) return } err = handler.bbs.ResolveTask(task) if err != nil { logger.Error("handler.resolve-failed", map[string]interface{}{ "task": task.Guid, "error": err.Error(), }) return } logger.Info("handler.resolved", map[string]interface{}{ "task": task.Guid, }) }
// The executor calls this when it has finished running the runonce (be it success or failure) // stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?) // This really really shouldn't fail. If it does, blog about it and walk away. If it failed in a // consistent way (i.e. key already exists), there's probably a flaw in our design. func (self *executorBBS) CompleteTask(task *models.Task, failed bool, failureReason string, result string) error { originalValue := task.ToJSON() task.UpdatedAt = self.timeProvider.Time().UnixNano() task.State = models.TaskStateCompleted task.Failed = failed task.FailureReason = failureReason task.Result = result return retryIndefinitelyOnStoreTimeout(func() error { err := self.store.CompareAndSwap(storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: originalValue, }, storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: task.ToJSON(), }) if err != nil { return err } self.kicker.Complete(task) return nil }) }
func (s *stagerBBS) ResolvingTask(task *models.Task) error { originalValue := task.ToJSON() task.UpdatedAt = s.timeProvider.Time().UnixNano() task.State = models.TaskStateResolving return retryIndefinitelyOnStoreTimeout(func() error { return s.store.CompareAndSwap(storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: originalValue, }, storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: task.ToJSON(), }) }) }
// The executor calls this when it is about to run the runonce in the claimed container // stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?) // If this fails, the executor should assume that someone else is running and should clean up and bail func (self *executorBBS) StartTask(task *models.Task, containerHandle string) error { originalValue := task.ToJSON() task.UpdatedAt = self.timeProvider.Time().UnixNano() task.State = models.TaskStateRunning task.ContainerHandle = containerHandle return retryIndefinitelyOnStoreTimeout(func() error { return self.store.CompareAndSwap(storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: originalValue, }, storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: task.ToJSON(), }) }) }
// The executor calls this when it wants to claim a runonce // stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?) // If this fails, the executor should assume that someone else is handling the claim and should bail func (self *executorBBS) ClaimTask(task *models.Task, executorID string) error { originalValue := task.ToJSON() task.UpdatedAt = self.timeProvider.Time().UnixNano() task.State = models.TaskStateClaimed task.ExecutorID = executorID return retryIndefinitelyOnStoreTimeout(func() error { return self.store.CompareAndSwap(storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: originalValue, }, storeadapter.StoreNode{ Key: taskSchemaPath(task), Value: task.ToJSON(), }) }) }
// The stager calls this when it wants to desire a payload // stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?) // If this fails, the stager should bail and run its "this-failed-to-stage" routine func (s *stagerBBS) DesireTask(task *models.Task) error { return retryIndefinitelyOnStoreTimeout(func() error { if task.CreatedAt == 0 { task.CreatedAt = s.timeProvider.Time().UnixNano() } task.UpdatedAt = s.timeProvider.Time().UnixNano() task.State = models.TaskStatePending err := s.store.SetMulti([]storeadapter.StoreNode{ { Key: taskSchemaPath(task), Value: task.ToJSON(), }, }) if err != nil { return err } s.kicker.Desire(task) return nil }) }