func testTaskRunner(restarts bool) (*MockTaskStateUpdater, *TaskRunner) { logger := testLogger() conf := DefaultConfig() conf.StateDir = os.TempDir() conf.AllocDir = os.TempDir() upd := &MockTaskStateUpdater{} alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] consulClient, _ := NewConsulService(&consulServiceConfig{logger, "127.0.0.1:8500", "", "", false, false, &structs.Node{}}) // Initialize the port listing. This should be done by the offer process but // we have a mock so that doesn't happen. task.Resources.Networks[0].ReservedPorts = []structs.Port{{"", 80}} allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID)) allocDir.Build([]*structs.Task{task}) ctx := driver.NewExecContext(allocDir, alloc.ID) rp := structs.NewRestartPolicy(structs.JobTypeService) restartTracker := newRestartTracker(rp) if !restarts { restartTracker = noRestartsTracker() } state := alloc.TaskStates[task.Name] tr := NewTaskRunner(logger, conf, upd.Update, ctx, mock.Alloc(), task, state, restartTracker, consulClient) return upd, tr }
func (c *Builder) CreateNomadJob(pipeline *structs.Pipeline, runId int) (*NomadJob, error) { config := make(map[string]interface{}) config["container"] = pipeline.Container config["pipeline"] = pipeline.Name config["run_id"] = strconv.Itoa(runId) config["server_url"] = c.ServerURL resources := &nomadStructs.Resources{ CPU: 1024, MemoryMB: 128, } task := &nomadStructs.Task{ Name: pipeline.Name, Driver: "gypsy", Config: config, Resources: resources, } group := &nomadStructs.TaskGroup{ Name: pipeline.Name, Count: 1, Tasks: []*nomadStructs.Task{task}, RestartPolicy: nomadStructs.NewRestartPolicy("batch"), } job := &nomadStructs.Job{ ID: pipeline.Name, Name: pipeline.Name, Region: "global", Priority: 50, Datacenters: []string{"dc1"}, Type: "batch", TaskGroups: []*nomadStructs.TaskGroup{group}, } if err := job.Validate(); err != nil { log.Errorf("Nomad job validation failed. Error: %s\n", err) return nil, err } apiJob, err := convertJob(job) if err != nil { log.Errorf("Failed to convert nomad job in api call. Error: %s\n", err) return nil, err } nomadConfig := nomadApi.DefaultConfig() nomadClient, err := nomadApi.NewClient(nomadConfig) if err != nil { log.Errorf("Error creating nomad api client: %s", err) return nil, fmt.Errorf(fmt.Sprintf("Error creating nomad api client: %s", err)) } evalId, _, nomadErr := nomadClient.Jobs().Register(apiJob, nil) if nomadErr != nil { log.Errorf("Error submitting job: %s", nomadErr) return nil, fmt.Errorf(fmt.Sprintf("Error submitting job: %s", nomadErr)) } log.Infof("Syccessfullt submitted nomad job. Eval id: %s\n", evalId) return &NomadJob{ Pipeline: pipeline, Job: job, }, nil }
func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) { logger := testLogger() conf := DefaultConfig() conf.StateDir = os.TempDir() conf.AllocDir = os.TempDir() upd := &MockTaskStateUpdater{} alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] // Initialize the port listing. This should be done by the offer process but // we have a mock so that doesn't happen. task.Resources.Networks[0].ReservedPorts = []int{80} allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID)) allocDir.Build([]*structs.Task{task}) ctx := driver.NewExecContext(allocDir, alloc.ID) rp := structs.NewRestartPolicy(structs.JobTypeService) restartTracker := newRestartTracker(structs.JobTypeService, rp) tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc.ID, task, restartTracker) return upd, tr }
func parseJob(result *structs.Job, obj *hclobj.Object) error { if obj.Len() > 1 { return fmt.Errorf("only one 'job' block allowed") } // Get our job object obj = obj.Elem(true)[0] // Decode the full thing into a map[string]interface for ease var m map[string]interface{} if err := hcl.DecodeObject(&m, obj); err != nil { return err } delete(m, "constraint") delete(m, "meta") delete(m, "update") // Set the ID and name to the object key result.ID = obj.Key result.Name = obj.Key // Defaults result.Priority = 50 result.Region = "global" result.Type = "service" // Decode the rest if err := mapstructure.WeakDecode(m, result); err != nil { return err } // Parse constraints if o := obj.Get("constraint", false); o != nil { if err := parseConstraints(&result.Constraints, o); err != nil { return err } } // If we have an update strategy, then parse that if o := obj.Get("update", false); o != nil { if err := parseUpdate(&result.Update, o); err != nil { return err } } // Parse out meta fields. These are in HCL as a list so we need // to iterate over them and merge them. if metaO := obj.Get("meta", false); metaO != nil { for _, o := range metaO.Elem(false) { var m map[string]interface{} if err := hcl.DecodeObject(&m, o); err != nil { return err } if err := mapstructure.WeakDecode(m, &result.Meta); err != nil { return err } } } // If we have tasks outside, create TaskGroups for them if o := obj.Get("task", false); o != nil { var tasks []*structs.Task if err := parseTasks(&tasks, o); err != nil { return err } result.TaskGroups = make([]*structs.TaskGroup, len(tasks), len(tasks)*2) for i, t := range tasks { result.TaskGroups[i] = &structs.TaskGroup{ Name: t.Name, Count: 1, Tasks: []*structs.Task{t}, RestartPolicy: structs.NewRestartPolicy(result.Type), } } } // Parse the task groups if o := obj.Get("group", false); o != nil { if err := parseGroups(result, o); err != nil { return fmt.Errorf("error parsing 'group': %s", err) } } return nil }
func parseGroups(result *structs.Job, obj *hclobj.Object) error { // Get all the maps of keys to the actual object objects := make(map[string]*hclobj.Object) for _, o1 := range obj.Elem(false) { for _, o2 := range o1.Elem(true) { if _, ok := objects[o2.Key]; ok { return fmt.Errorf( "group '%s' defined more than once", o2.Key) } objects[o2.Key] = o2 } } if len(objects) == 0 { return nil } // Go through each object and turn it into an actual result. collection := make([]*structs.TaskGroup, 0, len(objects)) for n, o := range objects { var m map[string]interface{} if err := hcl.DecodeObject(&m, o); err != nil { return err } delete(m, "constraint") delete(m, "meta") delete(m, "task") delete(m, "restart") // Default count to 1 if not specified if _, ok := m["count"]; !ok { m["count"] = 1 } // Build the group with the basic decode var g structs.TaskGroup g.Name = n if err := mapstructure.WeakDecode(m, &g); err != nil { return err } // Parse constraints if o := o.Get("constraint", false); o != nil { if err := parseConstraints(&g.Constraints, o); err != nil { return err } } g.RestartPolicy = structs.NewRestartPolicy(result.Type) if err := parseRestartPolicy(g.RestartPolicy, o); err != nil { return err } // Parse out meta fields. These are in HCL as a list so we need // to iterate over them and merge them. if metaO := o.Get("meta", false); metaO != nil { for _, o := range metaO.Elem(false) { var m map[string]interface{} if err := hcl.DecodeObject(&m, o); err != nil { return err } if err := mapstructure.WeakDecode(m, &g.Meta); err != nil { return err } } } // Parse tasks if o := o.Get("task", false); o != nil { if err := parseTasks(&g.Tasks, o); err != nil { return err } } collection = append(collection, &g) } result.TaskGroups = append(result.TaskGroups, collection...) return nil }
func parseJob(result *structs.Job, list *ast.ObjectList) error { list = list.Children() if len(list.Items) != 1 { return fmt.Errorf("only one 'job' block allowed") } // Get our job object obj := list.Items[0] // Decode the full thing into a map[string]interface for ease var m map[string]interface{} if err := hcl.DecodeObject(&m, obj.Val); err != nil { return err } delete(m, "constraint") delete(m, "meta") delete(m, "update") delete(m, "periodic") // Set the ID and name to the object key result.ID = obj.Keys[0].Token.Value().(string) result.Name = result.ID // Defaults result.Priority = 50 result.Region = "global" result.Type = "service" // Decode the rest if err := mapstructure.WeakDecode(m, result); err != nil { return err } // Value should be an object var listVal *ast.ObjectList if ot, ok := obj.Val.(*ast.ObjectType); ok { listVal = ot.List } else { return fmt.Errorf("job '%s' value: should be an object", result.ID) } // Parse constraints if o := listVal.Filter("constraint"); len(o.Items) > 0 { if err := parseConstraints(&result.Constraints, o); err != nil { return err } } // If we have an update strategy, then parse that if o := listVal.Filter("update"); len(o.Items) > 0 { if err := parseUpdate(&result.Update, o); err != nil { return err } } // If we have a periodic definition, then parse that if o := listVal.Filter("periodic"); len(o.Items) > 0 { if err := parsePeriodic(&result.Periodic, o); err != nil { return err } } // Parse out meta fields. These are in HCL as a list so we need // to iterate over them and merge them. if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { for _, o := range metaO.Elem().Items { var m map[string]interface{} if err := hcl.DecodeObject(&m, o.Val); err != nil { return err } if err := mapstructure.WeakDecode(m, &result.Meta); err != nil { return err } } } // If we have tasks outside, create TaskGroups for them if o := listVal.Filter("task"); len(o.Items) > 0 { var tasks []*structs.Task if err := parseTasks(result.Name, "", &tasks, o); err != nil { return err } result.TaskGroups = make([]*structs.TaskGroup, len(tasks), len(tasks)*2) for i, t := range tasks { result.TaskGroups[i] = &structs.TaskGroup{ Name: t.Name, Count: 1, Tasks: []*structs.Task{t}, RestartPolicy: structs.NewRestartPolicy(result.Type), } } } // Parse the task groups if o := listVal.Filter("group"); len(o.Items) > 0 { if err := parseGroups(result, o); err != nil { return fmt.Errorf("error parsing 'group': %s", err) } } return nil }
func parseGroups(result *structs.Job, list *ast.ObjectList) error { list = list.Children() if len(list.Items) == 0 { return nil } // Go through each object and turn it into an actual result. collection := make([]*structs.TaskGroup, 0, len(list.Items)) seen := make(map[string]struct{}) for _, item := range list.Items { n := item.Keys[0].Token.Value().(string) // Make sure we haven't already found this if _, ok := seen[n]; ok { return fmt.Errorf("group '%s' defined more than once", n) } seen[n] = struct{}{} // We need this later var listVal *ast.ObjectList if ot, ok := item.Val.(*ast.ObjectType); ok { listVal = ot.List } else { return fmt.Errorf("group '%s': should be an object", n) } var m map[string]interface{} if err := hcl.DecodeObject(&m, item.Val); err != nil { return err } delete(m, "constraint") delete(m, "meta") delete(m, "task") delete(m, "restart") // Default count to 1 if not specified if _, ok := m["count"]; !ok { m["count"] = 1 } // Build the group with the basic decode var g structs.TaskGroup g.Name = n if err := mapstructure.WeakDecode(m, &g); err != nil { return err } // Parse constraints if o := listVal.Filter("constraint"); len(o.Items) > 0 { if err := parseConstraints(&g.Constraints, o); err != nil { return err } } g.RestartPolicy = structs.NewRestartPolicy(result.Type) // Parse restart policy if o := listVal.Filter("restart"); len(o.Items) > 0 { if err := parseRestartPolicy(g.RestartPolicy, o); err != nil { return err } } // Parse out meta fields. These are in HCL as a list so we need // to iterate over them and merge them. if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { for _, o := range metaO.Elem().Items { var m map[string]interface{} if err := hcl.DecodeObject(&m, o.Val); err != nil { return err } if err := mapstructure.WeakDecode(m, &g.Meta); err != nil { return err } } } // Parse tasks if o := listVal.Filter("task"); len(o.Items) > 0 { if err := parseTasks(result.Name, g.Name, &g.Tasks, o); err != nil { return err } } collection = append(collection, &g) } result.TaskGroups = append(result.TaskGroups, collection...) return nil }