// registerService registers a Service with Consul
func (c *ConsulService) registerService(service *structs.Service, task *structs.Task, alloc *structs.Allocation) error {
var mErr multierror.Error
host, port := task.FindHostAndPortFor(service.PortLabel)
if host == "" || port == 0 {
return fmt.Errorf("consul: the port:%q marked for registration of service: %q couldn't be found", service.PortLabel, service.Name)
}
serviceID := alloc.Services[service.Name]
c.serviceStates[serviceID] = service.Hash()
asr := &consul.AgentServiceRegistration{
ID: serviceID,
Name: service.Name,
Tags: service.Tags,
Port: port,
Address: host,
}
if err := c.client.ServiceRegister(asr); err != nil {
c.printLogMessage("[DEBUG] consul: error while registering service %v with consul: %v", service.Name, err)
mErr.Errors = append(mErr.Errors, err)
}
for _, check := range service.Checks {
cr := c.makeCheck(serviceID, check, host, port)
if err := c.registerCheck(cr); err != nil {
c.printLogMessage("[DEBUG] consul: error while registering check %v with consul: %v", check.Name, err)
mErr.Errors = append(mErr.Errors, err)
}
}
return mErr.ErrorOrNil()
}
// registerService registers a Service with Consul
func (c *ConsulService) registerService(service *structs.Service, task *structs.Task, allocID string) error {
var mErr multierror.Error
service.Id = fmt.Sprintf("%s-%s", allocID, service.Name)
host, port := task.FindHostAndPortFor(service.PortLabel)
if host == "" || port == 0 {
return fmt.Errorf("consul: The port:%s marked for registration of service: %s couldn't be found", service.PortLabel, service.Name)
}
c.serviceStates[service.Id] = service.Hash()
asr := &consul.AgentServiceRegistration{
ID: service.Id,
Name: service.Name,
Tags: service.Tags,
Port: port,
Address: host,
}
if err := c.client.ServiceRegister(asr); err != nil {
c.logger.Printf("[DEBUG] consul: Error while registering service %v with Consul: %v", service.Name, err)
mErr.Errors = append(mErr.Errors, err)
}
for _, check := range service.Checks {
cr := c.makeCheck(service, check, host, port)
if err := c.registerCheck(cr); err != nil {
c.logger.Printf("[ERROR] consul: Error while registerting check %v with Consul: %v", check.Name, err)
mErr.Errors = append(mErr.Errors, err)
}
}
return mErr.ErrorOrNil()
}
func TestConsul_Services_Deleted_From_Task(t *testing.T) {
c := newConsulService()
task := structs.Task{
Name: "redis",
Services: []*structs.Service{
&structs.Service{
Name: "example-cache-redis",
Tags: []string{"global"},
PortLabel: "db",
},
},
Resources: &structs.Resources{
Networks: []*structs.NetworkResource{
{
IP: "10.10.0.1",
DynamicPorts: []structs.Port{{"db", 20413}},
},
},
},
}
c.Register(&task, "1")
if len(c.serviceStates) != 1 {
t.Fatalf("Expected tracked services: %v, Actual: %v", 1, len(c.serviceStates))
}
task.Services = []*structs.Service{}
c.performSync()
if len(c.serviceStates) != 0 {
t.Fatalf("Expected tracked services: %v, Actual: %v", 0, len(c.serviceStates))
}
}
// handleUpdate takes an updated allocation and updates internal state to
// reflect the new config for the task.
func (r *TaskRunner) handleUpdate(update *structs.Allocation) error {
// Extract the task group from the alloc.
tg := update.Job.LookupTaskGroup(update.TaskGroup)
if tg == nil {
return fmt.Errorf("alloc '%s' missing task group '%s'", update.ID, update.TaskGroup)
}
// Extract the task.
var updatedTask *structs.Task
for _, t := range tg.Tasks {
if t.Name == r.task.Name {
updatedTask = t
}
}
if updatedTask == nil {
return fmt.Errorf("task group %q doesn't contain task %q", tg.Name, r.task.Name)
}
// Merge in the task resources
updatedTask.Resources = update.TaskResources[updatedTask.Name]
// Update will update resources and store the new kill timeout.
var mErr multierror.Error
r.handleLock.Lock()
if r.handle != nil {
if err := r.handle.Update(updatedTask); err != nil {
mErr.Errors = append(mErr.Errors, fmt.Errorf("updating task resources failed: %v", err))
}
}
r.handleLock.Unlock()
// Update the restart policy.
if r.restartTracker != nil {
r.restartTracker.SetPolicy(tg.RestartPolicy)
}
// Hash services returns the hash of the task's services
hashServices := func(task *structs.Task) uint64 {
h, err := hashstructure.Hash(task.Services, nil)
if err != nil {
mErr.Errors = append(mErr.Errors, fmt.Errorf("hashing services failed %#v: %v", task.Services, err))
}
return h
}
// Re-register the task to consul if any of the services have changed.
if hashServices(updatedTask) != hashServices(r.task) {
if err := r.consulService.Register(updatedTask, update); err != nil {
mErr.Errors = append(mErr.Errors, fmt.Errorf("updating services with consul failed: %v", err))
}
}
// Store the updated alloc.
r.alloc = update
r.task = updatedTask
return mErr.ErrorOrNil()
}
// NewTaskRunner is used to create a new task context
func NewTaskRunner(logger *log.Logger, config *config.Config,
updater TaskStateUpdater, ctx *driver.ExecContext,
alloc *structs.Allocation, task *structs.Task,
consulService *ConsulService) *TaskRunner {
// Merge in the task resources
task.Resources = alloc.TaskResources[task.Name]
// Build the restart tracker.
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
if tg == nil {
logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup)
return nil
}
restartTracker := newRestartTracker(tg.RestartPolicy, alloc.Job.Type)
tc := &TaskRunner{
config: config,
updater: updater,
logger: logger,
restartTracker: restartTracker,
consulService: consulService,
ctx: ctx,
alloc: alloc,
task: task,
updateCh: make(chan *structs.Allocation, 8),
destroyCh: make(chan struct{}),
waitCh: make(chan struct{}),
}
return tc
}
func parseServices(jobName string, taskGroupName string, task *structs.Task, serviceObjs *ast.ObjectList) error {
task.Services = make([]*structs.Service, len(serviceObjs.Items))
var defaultServiceName bool
for idx, o := range serviceObjs.Items {
// Check for invalid keys
valid := []string{
"name",
"tags",
"port",
"check",
}
if err := checkHCLKeys(o.Val, valid); err != nil {
return multierror.Prefix(err, fmt.Sprintf("service (%d) ->", idx))
}
var service structs.Service
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
delete(m, "check")
if err := mapstructure.WeakDecode(m, &service); err != nil {
return err
}
if defaultServiceName && service.Name == "" {
return fmt.Errorf("Only one service block may omit the Name field")
}
if service.Name == "" {
defaultServiceName = true
service.Name = fmt.Sprintf("%s-%s-%s", jobName, taskGroupName, task.Name)
}
// Filter checks
var checkList *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
checkList = ot.List
} else {
return fmt.Errorf("service '%s': should be an object", service.Name)
}
if co := checkList.Filter("check"); len(co.Items) > 0 {
if err := parseChecks(&service, co); err != nil {
return multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name))
}
}
task.Services[idx] = &service
}
return nil
}
// handleUpdate takes an updated allocation and updates internal state to
// reflect the new config for the task.
func (r *TaskRunner) handleUpdate(update *structs.Allocation) error {
// Extract the task group from the alloc.
tg := update.Job.LookupTaskGroup(update.TaskGroup)
if tg == nil {
return fmt.Errorf("alloc '%s' missing task group '%s'", update.ID, update.TaskGroup)
}
// Extract the task.
var updatedTask *structs.Task
for _, t := range tg.Tasks {
if t.Name == r.task.Name {
updatedTask = t
}
}
if updatedTask == nil {
return fmt.Errorf("task group %q doesn't contain task %q", tg.Name, r.task.Name)
}
// Merge in the task resources
updatedTask.Resources = update.TaskResources[updatedTask.Name]
// Update will update resources and store the new kill timeout.
var mErr multierror.Error
r.handleLock.Lock()
if r.handle != nil {
if err := r.handle.Update(updatedTask); err != nil {
mErr.Errors = append(mErr.Errors, fmt.Errorf("updating task resources failed: %v", err))
}
}
r.handleLock.Unlock()
// Update the restart policy.
if r.restartTracker != nil {
r.restartTracker.SetPolicy(tg.RestartPolicy)
}
// Store the updated alloc.
r.alloc = update
r.task = updatedTask
return mErr.ErrorOrNil()
}
func TestTaskRunner_Update(t *testing.T) {
ctestutil.ExecCompatible(t)
_, tr := testTaskRunner()
// Change command to ensure we run for a bit
tr.task.Config["command"] = "/bin/sleep"
tr.task.Config["args"] = "10"
go tr.Run()
defer tr.Destroy()
defer tr.ctx.AllocDir.Destroy()
// Update the task definition
newTask := new(structs.Task)
*newTask = *tr.task
newTask.Driver = "foobar"
tr.Update(newTask)
// Wait for update to take place
testutil.WaitForResult(func() (bool, error) {
return tr.task == newTask, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
// NewTaskRunner is used to create a new task context
func NewTaskRunner(logger *log.Logger, config *config.Config,
updater TaskStateUpdater, ctx *driver.ExecContext,
alloc *structs.Allocation, task *structs.Task,
vaultClient vaultclient.VaultClient) *TaskRunner {
// Merge in the task resources
task.Resources = alloc.TaskResources[task.Name]
// Build the restart tracker.
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
if tg == nil {
logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup)
return nil
}
restartTracker := newRestartTracker(tg.RestartPolicy, alloc.Job.Type)
// Get the task directory
taskDir, ok := ctx.AllocDir.TaskDirs[task.Name]
if !ok {
logger.Printf("[ERR] client: task directory for alloc %q task %q couldn't be found", alloc.ID, task.Name)
return nil
}
tc := &TaskRunner{
config: config,
updater: updater,
logger: logger,
restartTracker: restartTracker,
ctx: ctx,
alloc: alloc,
task: task,
taskDir: taskDir,
vaultClient: vaultClient,
vaultFuture: NewTokenFuture().Set(""),
updateCh: make(chan *structs.Allocation, 64),
destroyCh: make(chan struct{}),
waitCh: make(chan struct{}),
startCh: make(chan struct{}, 1),
unblockCh: make(chan struct{}),
restartCh: make(chan *structs.TaskEvent),
signalCh: make(chan SignalEvent),
}
return tc
}
// NewTaskRunner is used to create a new task context
func NewTaskRunner(logger *log.Logger, config *config.Config,
updater TaskStateUpdater, ctx *driver.ExecContext,
alloc *structs.Allocation, task *structs.Task) *TaskRunner {
// Merge in the task resources
task.Resources = alloc.TaskResources[task.Name]
// Build the restart tracker.
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
if tg == nil {
logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup)
return nil
}
restartTracker := newRestartTracker(tg.RestartPolicy, alloc.Job.Type)
resourceUsage, err := stats.NewRingBuff(config.StatsDataPoints)
if err != nil {
logger.Printf("[ERR] client: can't create resource usage buffer: %v", err)
return nil
}
tc := &TaskRunner{
config: config,
updater: updater,
logger: logger,
restartTracker: restartTracker,
resourceUsage: resourceUsage,
ctx: ctx,
alloc: alloc,
task: task,
updateCh: make(chan *structs.Allocation, 64),
destroyCh: make(chan struct{}),
waitCh: make(chan struct{}),
}
return tc
}
func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, list *ast.ObjectList) error {
list = list.Children()
if len(list.Items) == 0 {
return nil
}
// Go through each object and turn it into an actual result.
seen := make(map[string]struct{})
for _, item := range list.Items {
n := item.Keys[0].Token.Value().(string)
// Make sure we haven't already found this
if _, ok := seen[n]; ok {
return fmt.Errorf("task '%s' defined more than once", n)
}
seen[n] = struct{}{}
// We need this later
var listVal *ast.ObjectList
if ot, ok := item.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("group '%s': should be an object", n)
}
// Check for invalid keys
valid := []string{
"artifact",
"config",
"constraint",
"driver",
"env",
"kill_timeout",
"logs",
"meta",
"resources",
"service",
"user",
"vault",
}
if err := checkHCLKeys(listVal, valid); err != nil {
return multierror.Prefix(err, fmt.Sprintf("'%s' ->", n))
}
var m map[string]interface{}
if err := hcl.DecodeObject(&m, item.Val); err != nil {
return err
}
delete(m, "artifact")
delete(m, "config")
delete(m, "constraint")
delete(m, "env")
delete(m, "logs")
delete(m, "meta")
delete(m, "resources")
delete(m, "service")
delete(m, "vault")
// Build the task
var t structs.Task
t.Name = n
if taskGroupName == "" {
taskGroupName = n
}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: &t,
})
if err != nil {
return err
}
if err := dec.Decode(m); err != nil {
return err
}
// If we have env, then parse them
if o := listVal.Filter("env"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Env); err != nil {
return err
}
}
}
if o := listVal.Filter("service"); len(o.Items) > 0 {
if err := parseServices(jobName, taskGroupName, &t, o); err != nil {
return multierror.Prefix(err, fmt.Sprintf("'%s',", n))
}
}
// If we have config, then parse that
if o := listVal.Filter("config"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Config); err != nil {
return err
}
}
// Instantiate a driver to validate the configuration
d, err := driver.NewDriver(
t.Driver,
driver.NewEmptyDriverContext(),
)
if err != nil {
return multierror.Prefix(err,
fmt.Sprintf("'%s', config ->", n))
}
if err := d.Validate(t.Config); err != nil {
return multierror.Prefix(err,
fmt.Sprintf("'%s', config ->", n))
}
}
// Parse constraints
if o := listVal.Filter("constraint"); len(o.Items) > 0 {
if err := parseConstraints(&t.Constraints, o); err != nil {
return multierror.Prefix(err, fmt.Sprintf(
"'%s', constraint ->", n))
}
}
// Parse out meta fields. These are in HCL as a list so we need
// to iterate over them and merge them.
if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 {
for _, o := range metaO.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Meta); err != nil {
return err
}
}
}
// If we have resources, then parse that
if o := listVal.Filter("resources"); len(o.Items) > 0 {
var r structs.Resources
if err := parseResources(&r, o); err != nil {
return multierror.Prefix(err, fmt.Sprintf("'%s',", n))
}
t.Resources = &r
}
// If we have logs then parse that
logConfig := structs.DefaultLogConfig()
if o := listVal.Filter("logs"); len(o.Items) > 0 {
if len(o.Items) > 1 {
return fmt.Errorf("only one logs block is allowed in a Task. Number of logs block found: %d", len(o.Items))
}
var m map[string]interface{}
logsBlock := o.Items[0]
// Check for invalid keys
valid := []string{
"max_files",
"max_file_size",
}
if err := checkHCLKeys(logsBlock.Val, valid); err != nil {
return multierror.Prefix(err, fmt.Sprintf("'%s', logs ->", n))
}
if err := hcl.DecodeObject(&m, logsBlock.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &logConfig); err != nil {
return err
}
}
t.LogConfig = logConfig
// Parse artifacts
if o := listVal.Filter("artifact"); len(o.Items) > 0 {
if err := parseArtifacts(&t.Artifacts, o); err != nil {
return multierror.Prefix(err, fmt.Sprintf("'%s', artifact ->", n))
}
}
// If we have a vault block, then parse that
if o := listVal.Filter("vault"); len(o.Items) > 0 {
var v structs.Vault
if err := parseVault(&v, o); err != nil {
return multierror.Prefix(err, fmt.Sprintf("'%s', vault ->", n))
}
t.Vault = &v
}
*result = append(*result, &t)
}
return nil
}
func parseTasks(result *[]*structs.Task, obj *hclobj.Object) error {
// Get all the maps of keys to the actual object
objects := make([]*hclobj.Object, 0, 5)
set := make(map[string]struct{})
for _, o1 := range obj.Elem(false) {
for _, o2 := range o1.Elem(true) {
if _, ok := set[o2.Key]; ok {
return fmt.Errorf(
"group '%s' defined more than once",
o2.Key)
}
objects = append(objects, o2)
set[o2.Key] = struct{}{}
}
}
if len(objects) == 0 {
return nil
}
for _, o := range objects {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
return err
}
delete(m, "config")
delete(m, "constraint")
delete(m, "meta")
delete(m, "resources")
// Build the task
var t structs.Task
t.Name = o.Key
if err := mapstructure.WeakDecode(m, &t); err != nil {
return err
}
// If we have config, then parse that
if o := o.Get("config", false); o != nil {
for _, o := range o.Elem(false) {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Config); err != nil {
return err
}
}
}
// Parse constraints
if o := o.Get("constraint", false); o != nil {
if err := parseConstraints(&t.Constraints, o); err != nil {
return err
}
}
// Parse out meta fields. These are in HCL as a list so we need
// to iterate over them and merge them.
if metaO := o.Get("meta", false); metaO != nil {
for _, o := range metaO.Elem(false) {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Meta); err != nil {
return err
}
}
}
// If we have resources, then parse that
if o := o.Get("resources", false); o != nil {
var r structs.Resources
if err := parseResources(&r, o); err != nil {
return fmt.Errorf("task '%s': %s", t.Name, err)
}
t.Resources = &r
}
*result = append(*result, &t)
}
return nil
}
func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, list *ast.ObjectList) error {
list = list.Children()
if len(list.Items) == 0 {
return nil
}
// Go through each object and turn it into an actual result.
seen := make(map[string]struct{})
for _, item := range list.Items {
n := item.Keys[0].Token.Value().(string)
// Make sure we haven't already found this
if _, ok := seen[n]; ok {
return fmt.Errorf("task '%s' defined more than once", n)
}
seen[n] = struct{}{}
// We need this later
var listVal *ast.ObjectList
if ot, ok := item.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("group '%s': should be an object", n)
}
var m map[string]interface{}
if err := hcl.DecodeObject(&m, item.Val); err != nil {
return err
}
delete(m, "config")
delete(m, "env")
delete(m, "constraint")
delete(m, "service")
delete(m, "meta")
delete(m, "resources")
// Build the task
var t structs.Task
t.Name = n
if taskGroupName == "" {
taskGroupName = n
}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
Result: &t,
})
if err != nil {
return err
}
if err := dec.Decode(m); err != nil {
return err
}
// If we have env, then parse them
if o := listVal.Filter("env"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Env); err != nil {
return err
}
}
}
if o := listVal.Filter("service"); len(o.Items) > 0 {
if err := parseServices(jobName, taskGroupName, &t, o); err != nil {
return err
}
}
// If we have config, then parse that
if o := listVal.Filter("config"); len(o.Items) > 0 {
for _, o := range o.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Config); err != nil {
return err
}
}
}
// Parse constraints
if o := listVal.Filter("constraint"); len(o.Items) > 0 {
if err := parseConstraints(&t.Constraints, o); err != nil {
return err
}
}
// Parse out meta fields. These are in HCL as a list so we need
// to iterate over them and merge them.
if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 {
for _, o := range metaO.Elem().Items {
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &t.Meta); err != nil {
return err
}
}
}
// If we have resources, then parse that
if o := listVal.Filter("resources"); len(o.Items) > 0 {
var r structs.Resources
if err := parseResources(&r, o); err != nil {
return fmt.Errorf("task '%s': %s", t.Name, err)
}
t.Resources = &r
}
*result = append(*result, &t)
}
return nil
}
// SetTaskEnv configures the environment of t so that Run executes a testtask
// script when called from within t.
func SetTaskEnv(t *structs.Task) {
if t.Env == nil {
t.Env = map[string]string{}
}
t.Env["TEST_TASK"] = "execute"
}