func testTaskRunner(restarts bool) (*MockTaskStateUpdater, *TaskRunner) { logger := testLogger() conf := DefaultConfig() conf.StateDir = os.TempDir() conf.AllocDir = os.TempDir() upd := &MockTaskStateUpdater{} alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] consulClient, _ := NewConsulService(&consulServiceConfig{logger, "127.0.0.1:8500", "", "", false, false, &structs.Node{}}) // Initialize the port listing. This should be done by the offer process but // we have a mock so that doesn't happen. task.Resources.Networks[0].ReservedPorts = []structs.Port{{"", 80}} allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID)) allocDir.Build([]*structs.Task{task}) ctx := driver.NewExecContext(allocDir, alloc.ID) rp := structs.NewRestartPolicy(structs.JobTypeService) restartTracker := newRestartTracker(rp) if !restarts { restartTracker = noRestartsTracker() } state := alloc.TaskStates[task.Name] tr := NewTaskRunner(logger, conf, upd.Update, ctx, mock.Alloc(), task, state, restartTracker, consulClient) return upd, tr }
func mockAllocDir(t *testing.T) (string, *allocdir.AllocDir) { alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] allocDir := allocdir.NewAllocDir(filepath.Join(os.TempDir(), alloc.ID)) if err := allocDir.Build([]*structs.Task{task}); err != nil { log.Panicf("allocDir.Build() failed: %v", err) } return task.Name, allocDir }
// tempAllocDir returns a new alloc dir that is rooted in a temp dir. The caller // should destroy the temp dir. func tempAllocDir(t *testing.T) *allocdir.AllocDir { dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("TempDir() failed: %v", err) } if err := os.Chmod(dir, 0777); err != nil { t.Fatalf("failed to chmod dir: %v", err) } return allocdir.NewAllocDir(dir) }
func setupDockerVolumes(t *testing.T, cfg *config.Config) (*structs.Task, Driver, *ExecContext, string, func()) { if !testutil.DockerIsConnected(t) { t.SkipNow() } tmpvol, err := ioutil.TempDir("", "nomadtest_dockerdriver_volumes") if err != nil { t.Fatalf("error creating temporary dir: %v", err) } randfn := fmt.Sprintf("test-%d", rand.Int()) hostpath := path.Join(tmpvol, randfn) contpath := path.Join("/mnt/vol", randfn) task := &structs.Task{ Name: "ls", Config: map[string]interface{}{ "image": "busybox", "load": []string{"busybox.tar"}, "command": "touch", "args": []string{contpath}, "volumes": []string{fmt.Sprintf("%s:/mnt/vol", tmpvol)}, }, LogConfig: &structs.LogConfig{ MaxFiles: 10, MaxFileSizeMB: 10, }, Resources: basicResources, } allocDir := allocdir.NewAllocDir(filepath.Join(cfg.AllocDir, structs.GenerateUUID()), task.Resources.DiskMB) allocDir.Build([]*structs.Task{task}) alloc := mock.Alloc() execCtx := NewExecContext(allocDir, alloc.ID) cleanup := func() { execCtx.AllocDir.Destroy() os.RemoveAll(tmpvol) } taskEnv, err := GetTaskEnv(allocDir, cfg.Node, task, alloc, "") if err != nil { cleanup() t.Fatalf("Failed to get task env: %v", err) } driverCtx := NewDriverContext(task.Name, cfg, cfg.Node, testLogger(), taskEnv) driver := NewDockerDriver(driverCtx) copyImage(execCtx, task, "busybox.tar", t) return task, driver, execCtx, hostpath, cleanup }
func testDriverContexts(task *structs.Task) (*DriverContext, *ExecContext) { cfg := testConfig() allocDir := allocdir.NewAllocDir(filepath.Join(cfg.AllocDir, structs.GenerateUUID())) allocDir.Build([]*structs.Task{task}) execCtx := NewExecContext(allocDir, fmt.Sprintf("alloc-id-%d", int(rand.Int31()))) taskEnv, err := GetTaskEnv(allocDir, cfg.Node, task) if err != nil { return nil, nil } driverCtx := NewDriverContext(task.Name, cfg, cfg.Node, testLogger(), taskEnv) return driverCtx, execCtx }
func setupDockerVolumes(t *testing.T, cfg *config.Config, hostpath string) (*structs.Task, Driver, *ExecContext, string, func()) { if !testutil.DockerIsConnected(t) { t.SkipNow() } randfn := fmt.Sprintf("test-%d", rand.Int()) hostfile := filepath.Join(hostpath, randfn) containerPath := "/mnt/vol" containerFile := filepath.Join(containerPath, randfn) task := &structs.Task{ Name: "ls", Env: map[string]string{"VOL_PATH": containerPath}, Config: map[string]interface{}{ "image": "busybox", "load": []string{"busybox.tar"}, "command": "touch", "args": []string{containerFile}, "volumes": []string{fmt.Sprintf("%s:${VOL_PATH}", hostpath)}, }, LogConfig: &structs.LogConfig{ MaxFiles: 10, MaxFileSizeMB: 10, }, Resources: basicResources, } allocDir := allocdir.NewAllocDir(filepath.Join(cfg.AllocDir, structs.GenerateUUID())) allocDir.Build([]*structs.Task{task}) alloc := mock.Alloc() execCtx := NewExecContext(allocDir, alloc.ID) cleanup := func() { execCtx.AllocDir.Destroy() if filepath.IsAbs(hostpath) { os.RemoveAll(hostpath) } } taskEnv, err := GetTaskEnv(allocDir, cfg.Node, task, alloc, "") if err != nil { cleanup() t.Fatalf("Failed to get task env: %v", err) } driverCtx := NewDriverContext(task.Name, cfg, cfg.Node, testLogger(), taskEnv) driver := NewDockerDriver(driverCtx) copyImage(execCtx, task, "busybox.tar", t) return task, driver, execCtx, hostfile, cleanup }
func testDriverContexts(task *structs.Task) (*DriverContext, *ExecContext) { cfg := testConfig() allocDir := allocdir.NewAllocDir(filepath.Join(cfg.AllocDir, structs.GenerateUUID())) allocDir.Build([]*structs.Task{task}) alloc := mock.Alloc() execCtx := NewExecContext(allocDir, alloc.ID) taskEnv, err := GetTaskEnv(allocDir, cfg.Node, task, alloc) if err != nil { return nil, nil } driverCtx := NewDriverContext(task.Name, cfg, cfg.Node, testLogger(), taskEnv) return driverCtx, execCtx }
func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) { logger := testLogger() conf := DefaultConfig() conf.StateDir = os.TempDir() conf.AllocDir = os.TempDir() upd := &MockTaskStateUpdater{} alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] // Initialize the port listing. This should be done by the offer process but // we have a mock so that doesn't happen. task.Resources.Networks[0].ReservedPorts = []int{80} allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID)) allocDir.Build([]*structs.Task{task}) ctx := driver.NewExecContext(allocDir) tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc.ID, task) return upd, tr }
// Creates a mock task runner using the first task in the first task group of // the passed allocation. func testTaskRunnerFromAlloc(restarts bool, alloc *structs.Allocation) (*MockTaskStateUpdater, *TaskRunner) { logger := testLogger() conf := DefaultConfig() conf.StateDir = os.TempDir() conf.AllocDir = os.TempDir() upd := &MockTaskStateUpdater{} task := alloc.Job.TaskGroups[0].Tasks[0] // Initialize the port listing. This should be done by the offer process but // we have a mock so that doesn't happen. task.Resources.Networks[0].ReservedPorts = []structs.Port{{"", 80}} allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID)) allocDir.Build([]*structs.Task{task}) ctx := driver.NewExecContext(allocDir, alloc.ID) tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc, task) if !restarts { tr.restartTracker = noRestartsTracker() } return upd, tr }
// Run is a long running goroutine used to manage an allocation func (r *AllocRunner) Run() { defer close(r.waitCh) go r.dirtySyncState() // Check if the allocation is in a terminal status alloc := r.alloc if alloc.TerminalStatus() { r.logger.Printf("[DEBUG] client: aborting runner for alloc '%s', terminal status", r.alloc.ID) return } r.logger.Printf("[DEBUG] client: starting runner for alloc '%s'", r.alloc.ID) // Find the task group to run in the allocation tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) if tg == nil { r.logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup) r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("missing task group '%s'", alloc.TaskGroup)) return } // Extract the RestartPolicy from the TG and set it on the alloc r.RestartPolicy = tg.RestartPolicy // Create the execution context if r.ctx == nil { allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) if err := allocDir.Build(tg.Tasks); err != nil { r.logger.Printf("[WARN] client: failed to build task directories: %v", err) r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup)) return } r.ctx = driver.NewExecContext(allocDir, r.alloc.ID) } // Start the task runners r.taskLock.Lock() for _, task := range tg.Tasks { // Skip tasks that were restored if _, ok := r.tasks[task.Name]; ok { continue } // Merge in the task resources task.Resources = alloc.TaskResources[task.Name] restartTracker := newRestartTracker(r.alloc.Job.Type, r.RestartPolicy) tr := NewTaskRunner(r.logger, r.config, r.setTaskStatus, r.ctx, r.alloc.ID, task, restartTracker) r.tasks[task.Name] = tr go tr.Run() } r.taskLock.Unlock() OUTER: // Wait for updates for { select { case update := <-r.updateCh: // Check if we're in a terminal status if update.TerminalStatus() { r.setAlloc(update) break OUTER } // Update the task groups r.taskLock.RLock() for _, task := range tg.Tasks { tr := r.tasks[task.Name] // Merge in the task resources task.Resources = update.TaskResources[task.Name] tr.Update(task) } r.taskLock.RUnlock() case <-r.destroyCh: break OUTER } } // Destroy each sub-task r.taskLock.RLock() defer r.taskLock.RUnlock() for _, tr := range r.tasks { tr.Destroy() } // Wait for termination of the task runners for _, tr := range r.tasks { <-tr.WaitCh() } // Final state sync r.retrySyncState(nil) // Check if we should destroy our state if r.destroy { if err := r.DestroyContext(); err != nil { r.logger.Printf("[ERR] client: failed to destroy context for alloc '%s': %v", r.alloc.ID, err) } if err := r.DestroyState(); err != nil { r.logger.Printf("[ERR] client: failed to destroy state for alloc '%s': %v", r.alloc.ID, err) } } r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID) }
func testDriverExecContext(task *structs.Task, driverCtx *DriverContext) *ExecContext { allocDir := allocdir.NewAllocDir(filepath.Join(driverCtx.config.AllocDir, structs.GenerateUUID())) allocDir.Build([]*structs.Task{task}) ctx := NewExecContext(allocDir, fmt.Sprintf("alloc-id-%d", int(rand.Int31()))) return ctx }
// Run is a long running goroutine used to manage an allocation func (r *AllocRunner) Run() { defer close(r.waitCh) go r.dirtySyncState() // Find the task group to run in the allocation alloc := r.alloc tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) if tg == nil { r.logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup) r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("missing task group '%s'", alloc.TaskGroup)) return } // Create the execution context r.ctxLock.Lock() if r.ctx == nil { allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) if err := allocDir.Build(tg.Tasks); err != nil { r.logger.Printf("[WARN] client: failed to build task directories: %v", err) r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup)) r.ctxLock.Unlock() return } r.ctx = driver.NewExecContext(allocDir, r.alloc.ID) } r.ctxLock.Unlock() // Check if the allocation is in a terminal status. In this case, we don't // start any of the task runners and directly wait for the destroy signal to // clean up the allocation. if alloc.TerminalStatus() { r.logger.Printf("[DEBUG] client: alloc %q in terminal status, waiting for destroy", r.alloc.ID) r.handleDestroy() r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID) return } // Start the task runners r.logger.Printf("[DEBUG] client: starting task runners for alloc '%s'", r.alloc.ID) r.taskLock.Lock() for _, task := range tg.Tasks { if _, ok := r.restored[task.Name]; ok { continue } tr := NewTaskRunner(r.logger, r.config, r.setTaskState, r.ctx, r.Alloc(), task.Copy()) r.tasks[task.Name] = tr tr.MarkReceived() go tr.Run() } r.taskLock.Unlock() OUTER: // Wait for updates for { select { case update := <-r.updateCh: // Store the updated allocation. r.allocLock.Lock() r.alloc = update r.allocLock.Unlock() // Check if we're in a terminal status if update.TerminalStatus() { break OUTER } // Update the task groups runners := r.getTaskRunners() for _, tr := range runners { tr.Update(update) } case <-r.destroyCh: break OUTER } } // Destroy each sub-task runners := r.getTaskRunners() for _, tr := range runners { tr.Destroy() } // Wait for termination of the task runners for _, tr := range runners { <-tr.WaitCh() } // Final state sync r.syncStatus() // Block until we should destroy the state of the alloc r.handleDestroy() r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID) }
// Run is a long running goroutine used to manage an allocation func (r *AllocRunner) Run() { defer close(r.waitCh) go r.dirtySyncState() // Find the task group to run in the allocation alloc := r.alloc tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) if tg == nil { r.logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup) r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("missing task group '%s'", alloc.TaskGroup)) return } // Create the execution context r.ctxLock.Lock() if r.ctx == nil { allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID), r.Alloc().Resources.DiskMB) if err := allocDir.Build(tg.Tasks); err != nil { r.logger.Printf("[WARN] client: failed to build task directories: %v", err) r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup)) r.ctxLock.Unlock() return } r.ctx = driver.NewExecContext(allocDir, r.alloc.ID) if r.otherAllocDir != nil { if err := allocDir.Move(r.otherAllocDir, tg.Tasks); err != nil { r.logger.Printf("[ERROR] client: failed to move alloc dir into alloc %q: %v", r.alloc.ID, err) } if err := r.otherAllocDir.Destroy(); err != nil { r.logger.Printf("[ERROR] client: error destroying allocdir %v", r.otherAllocDir.AllocDir, err) } } } r.ctxLock.Unlock() // Check if the allocation is in a terminal status. In this case, we don't // start any of the task runners and directly wait for the destroy signal to // clean up the allocation. if alloc.TerminalStatus() { r.logger.Printf("[DEBUG] client: alloc %q in terminal status, waiting for destroy", r.alloc.ID) r.handleDestroy() r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID) return } // Start the task runners r.logger.Printf("[DEBUG] client: starting task runners for alloc '%s'", r.alloc.ID) r.taskLock.Lock() for _, task := range tg.Tasks { if _, ok := r.restored[task.Name]; ok { continue } tr := NewTaskRunner(r.logger, r.config, r.setTaskState, r.ctx, r.Alloc(), task.Copy(), r.vaultClient) r.tasks[task.Name] = tr tr.MarkReceived() go tr.Run() } r.taskLock.Unlock() // Start watching the shared allocation directory for disk usage go r.ctx.AllocDir.StartDiskWatcher() watchdog := time.NewTicker(watchdogInterval) defer watchdog.Stop() // taskDestroyEvent contains an event that caused the destroyment of a task // in the allocation. var taskDestroyEvent *structs.TaskEvent OUTER: // Wait for updates for { select { case update := <-r.updateCh: // Store the updated allocation. r.allocLock.Lock() r.alloc = update r.allocLock.Unlock() // Check if we're in a terminal status if update.TerminalStatus() { taskDestroyEvent = structs.NewTaskEvent(structs.TaskKilled) break OUTER } // Update the task groups runners := r.getTaskRunners() for _, tr := range runners { tr.Update(update) } case <-watchdog.C: if event, desc := r.checkResources(); event != nil { r.setStatus(structs.AllocClientStatusFailed, desc) taskDestroyEvent = event break OUTER } case <-r.destroyCh: taskDestroyEvent = structs.NewTaskEvent(structs.TaskKilled) break OUTER } } // Kill the task runners r.destroyTaskRunners(taskDestroyEvent) // Stop watching the shared allocation directory r.ctx.AllocDir.StopDiskWatcher() // Block until we should destroy the state of the alloc r.handleDestroy() r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID) }
func testDriverExecContext(task *structs.Task, driverCtx *DriverContext) *ExecContext { allocDir := allocdir.NewAllocDir(filepath.Join(driverCtx.config.AllocDir, structs.GenerateUUID())) allocDir.Build([]*structs.Task{task}) ctx := NewExecContext(allocDir) return ctx }