func (handler *Handler) resolveTask(task *models.Task) {
err := handler.bbs.ResolvingTask(task)
if err != nil {
logger.Info("handler.resolving-failed", map[string]interface{}{
"task": task.Guid,
"error": err.Error(),
})
return
}
err = handler.natsClient.Publish(task.ReplyTo, task.ToJSON())
if err != nil {
logger.Error("handler.publish-failed", map[string]interface{}{
"task": task.Guid,
"error": err.Error(),
})
return
}
err = handler.bbs.ResolveTask(task)
if err != nil {
logger.Error("handler.resolve-failed", map[string]interface{}{
"task": task.Guid,
"error": err.Error(),
})
return
}
logger.Info("handler.resolved", map[string]interface{}{
"task": task.Guid,
})
}
func (kicker *HurlerKicker) Complete(task *models.Task) {
log.Println("kicking complete")
json := task.ToJSON()
res, err := http.DefaultClient.Do(&http.Request{
Method: "POST",
URL: &url.URL{
Scheme: "http",
Host: kicker.hurlerAddress,
Path: "/tasks",
},
Host: "stager",
Body: ioutil.NopCloser(bytes.NewBuffer(json)),
ContentLength: int64(len(json)),
Header: map[string][]string{
"Content-Type": []string{"application/json"},
},
})
if err != nil {
log.Println("kick complete error", err)
return
}
log.Println("kick complete", res.Status)
}
// The executor calls this when it has finished running the runonce (be it success or failure)
// stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// This really really shouldn't fail. If it does, blog about it and walk away. If it failed in a
// consistent way (i.e. key already exists), there's probably a flaw in our design.
func (self *executorBBS) CompleteTask(task *models.Task, failed bool, failureReason string, result string) error {
originalValue := task.ToJSON()
task.UpdatedAt = self.timeProvider.Time().UnixNano()
task.State = models.TaskStateCompleted
task.Failed = failed
task.FailureReason = failureReason
task.Result = result
return retryIndefinitelyOnStoreTimeout(func() error {
err := self.store.CompareAndSwap(storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: originalValue,
}, storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: task.ToJSON(),
})
if err != nil {
return err
}
self.kicker.Complete(task)
return nil
})
}
// The stager calls this when it wants to desire a payload
// stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the stager should bail and run its "this-failed-to-stage" routine
func (s *stagerBBS) DesireTask(task *models.Task) error {
return retryIndefinitelyOnStoreTimeout(func() error {
if task.CreatedAt == 0 {
task.CreatedAt = s.timeProvider.Time().UnixNano()
}
task.UpdatedAt = s.timeProvider.Time().UnixNano()
task.State = models.TaskStatePending
err := s.store.SetMulti([]storeadapter.StoreNode{
{
Key: taskSchemaPath(task),
Value: task.ToJSON(),
},
})
if err != nil {
return err
}
s.kicker.Desire(task)
return nil
})
}
// The executor calls this when it is about to run the runonce in the claimed container
// stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the executor should assume that someone else is running and should clean up and bail
func (self *executorBBS) StartTask(task *models.Task, containerHandle string) error {
originalValue := task.ToJSON()
task.UpdatedAt = self.timeProvider.Time().UnixNano()
task.State = models.TaskStateRunning
task.ContainerHandle = containerHandle
return retryIndefinitelyOnStoreTimeout(func() error {
return self.store.CompareAndSwap(storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: originalValue,
}, storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: task.ToJSON(),
})
})
}
// The executor calls this when it wants to claim a runonce
// stagerBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the executor should assume that someone else is handling the claim and should bail
func (self *executorBBS) ClaimTask(task *models.Task, executorID string) error {
originalValue := task.ToJSON()
task.UpdatedAt = self.timeProvider.Time().UnixNano()
task.State = models.TaskStateClaimed
task.ExecutorID = executorID
return retryIndefinitelyOnStoreTimeout(func() error {
return self.store.CompareAndSwap(storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: originalValue,
}, storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: task.ToJSON(),
})
})
}
func (s *stagerBBS) ResolvingTask(task *models.Task) error {
originalValue := task.ToJSON()
task.UpdatedAt = s.timeProvider.Time().UnixNano()
task.State = models.TaskStateResolving
return retryIndefinitelyOnStoreTimeout(func() error {
return s.store.CompareAndSwap(storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: originalValue,
}, storeadapter.StoreNode{
Key: taskSchemaPath(task),
Value: task.ToJSON(),
})
})
}
func demoteToCompleted(task models.Task) models.Task {
task.State = models.TaskStateCompleted
return task
}
func demoteToPending(task models.Task) models.Task {
task.State = models.TaskStatePending
task.ExecutorID = ""
task.ContainerHandle = ""
return task
}
func markTaskFailed(task models.Task, reason string) models.Task {
task.State = models.TaskStateCompleted
task.Failed = true
task.FailureReason = reason
return task
}
