func TestStreamFramer_Heartbeat(t *testing.T) {
// Create the stream framer
r, w := io.Pipe()
wrappedW := &WriteCloseChecker{WriteCloser: w}
hRate, bWindow := 100*time.Millisecond, 100*time.Millisecond
sf := NewStreamFramer(wrappedW, hRate, bWindow, 100)
sf.Run()
// Create a decoder
dec := codec.NewDecoder(r, jsonHandle)
// Start the reader
resultCh := make(chan struct{})
go func() {
for {
var frame StreamFrame
if err := dec.Decode(&frame); err != nil {
t.Fatalf("failed to decode")
}
if frame.IsHeartbeat() {
resultCh <- struct{}{}
return
}
}
}()
select {
case <-resultCh:
case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * hRate):
t.Fatalf("failed to heartbeat")
}
// Close the reader and wait. This should cause the runner to exit
if err := r.Close(); err != nil {
t.Fatalf("failed to close reader")
}
select {
case <-sf.ExitCh():
case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * hRate):
t.Fatalf("exit channel should close")
}
sf.Destroy()
if !wrappedW.Closed {
t.Fatalf("writer not closed")
}
}
func TestTaskTemplateManager_Unblock_Vault(t *testing.T) {
// Make a template that will render based on a key in Vault
vaultPath := "secret/password"
key := "password"
content := "barbaz"
embedded := fmt.Sprintf(`{{with secret "%s"}}{{.Data.%s}}{{end}}`, vaultPath, key)
file := "my.tmpl"
template := &structs.Template{
EmbeddedTmpl: embedded,
DestPath: file,
ChangeMode: structs.TemplateChangeModeNoop,
}
// Drop the retry rate
testRetryRate = 10 * time.Millisecond
harness := newTestHarness(t, []*structs.Template{template}, false, true)
harness.start(t)
defer harness.stop()
// Ensure no unblock
select {
case <-harness.mockHooks.UnblockCh:
t.Fatalf("Task unblock should not have been called")
case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
}
// Write the secret to Vault
logical := harness.vault.Client.Logical()
logical.Write(vaultPath, map[string]interface{}{key: content})
// Wait for the unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Task unblock should have been called")
}
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
if s := string(raw); s != content {
t.Fatalf("Unexpected template data; got %q, want %q", s, content)
}
}
func TestTaskTemplateManager_Signal_Error(t *testing.T) {
// Make a template that renders based on a key in Consul and sends SIGALRM
key1 := "foo"
content1 := "bar"
content2 := "baz"
embedded1 := fmt.Sprintf(`{{key "%s"}}`, key1)
file1 := "my.tmpl"
template := &structs.Template{
EmbeddedTmpl: embedded1,
DestPath: file1,
ChangeMode: structs.TemplateChangeModeSignal,
ChangeSignal: "SIGALRM",
}
// Drop the retry rate
testRetryRate = 10 * time.Millisecond
harness := newTestHarness(t, []*structs.Template{template}, true, false)
harness.start(t)
defer harness.stop()
harness.mockHooks.SignalError = fmt.Errorf("test error")
// Write the key to Consul
harness.consul.SetKV(key1, []byte(content1))
// Wait a little
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(2*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Should have received unblock: %+v", harness.mockHooks)
}
// Write the key to Consul
harness.consul.SetKV(key1, []byte(content2))
// Wait for kill channel
select {
case <-harness.mockHooks.KillCh:
break
case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Should have received a signals: %+v", harness.mockHooks)
}
if !strings.Contains(harness.mockHooks.KillReason, "Sending signals") {
t.Fatalf("Unexpected error", harness.mockHooks.KillReason)
}
}
func TestDockerDriver_VolumesEnabled(t *testing.T) {
cfg := testConfig()
tmpvol, err := ioutil.TempDir("", "nomadtest_docker_volumesenabled")
if err != nil {
t.Fatalf("error creating temporary dir: %v", err)
}
task, driver, execCtx, hostpath, cleanup := setupDockerVolumes(t, cfg, tmpvol)
defer cleanup()
handle, err := driver.Start(execCtx, task)
if err != nil {
t.Fatalf("Failed to start docker driver: %v", err)
}
defer handle.Kill()
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("unexpected err: %v", res)
}
case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
}
if _, err := ioutil.ReadFile(hostpath); err != nil {
t.Fatalf("unexpected error reading %s: %v", hostpath, err)
}
}
func TestVaultClient_EstablishConnection(t *testing.T) {
v := testutil.NewTestVault(t)
logger := log.New(os.Stderr, "TEST: ", log.Lshortfile|log.LstdFlags)
v.Config.ConnectionRetryIntv = 100 * time.Millisecond
v.Config.TaskTokenTTL = "10s"
c, err := NewVaultClient(v.Config, logger, nil)
if err != nil {
t.Fatalf("failed to build vault client: %v", err)
}
c.Start()
defer c.Stop()
// Sleep a little while and check that no connection has been established.
time.Sleep(100 * time.Duration(testutil.TestMultiplier()) * time.Millisecond)
if c.ConnectionEstablished() {
t.Fatalf("ConnectionEstablished() returned true before Vault server started")
}
// Start Vault
v.Start()
defer v.Stop()
testutil.WaitForResult(func() (bool, error) {
return c.ConnectionEstablished(), nil
}, func(err error) {
t.Fatalf("Connection not established")
})
}
func TestTaskTemplateManager_Interpolate_Destination(t *testing.T) {
// Make a template that will have its destination interpolated
content := "hello, world!"
file := "${node.unique.id}.tmpl"
template := &structs.Template{
EmbeddedTmpl: content,
DestPath: file,
ChangeMode: structs.TemplateChangeModeNoop,
}
harness := newTestHarness(t, []*structs.Template{template}, false, false)
harness.start(t)
defer harness.stop()
// Ensure unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Task unblock should have been called")
}
// Check the file is there
actual := fmt.Sprintf("%s.tmpl", harness.node.ID)
path := filepath.Join(harness.taskDir, actual)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
if s := string(raw); s != content {
t.Fatalf("Unexpected template data; got %q, want %q", s, content)
}
}
func TestTaskRunner_SimpleRun(t *testing.T) {
ctestutil.ExecCompatible(t)
upd, tr := testTaskRunner(false)
tr.MarkReceived()
go tr.Run()
defer tr.Destroy(structs.NewTaskEvent(structs.TaskKilled))
defer tr.ctx.AllocDir.Destroy()
select {
case <-tr.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
if len(upd.events) != 3 {
t.Fatalf("should have 3 updates: %#v", upd.events)
}
if upd.state != structs.TaskStateDead {
t.Fatalf("TaskState %v; want %v", upd.state, structs.TaskStateDead)
}
if upd.events[0].Type != structs.TaskReceived {
t.Fatalf("First Event was %v; want %v", upd.events[0].Type, structs.TaskReceived)
}
if upd.events[1].Type != structs.TaskStarted {
t.Fatalf("Second Event was %v; want %v", upd.events[1].Type, structs.TaskStarted)
}
if upd.events[2].Type != structs.TaskTerminated {
t.Fatalf("Third Event was %v; want %v", upd.events[2].Type, structs.TaskTerminated)
}
}
func TestTaskTemplateManager_Unblock_Static(t *testing.T) {
// Make a template that will render immediately
content := "hello, world!"
file := "my.tmpl"
template := &structs.Template{
EmbeddedTmpl: content,
DestPath: file,
ChangeMode: structs.TemplateChangeModeNoop,
}
harness := newTestHarness(t, []*structs.Template{template}, false, false)
harness.start(t)
defer harness.stop()
// Wait for the unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Task unblock should have been called")
}
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
if s := string(raw); s != content {
t.Fatalf("Unexpected template data; got %q, want %q", s, content)
}
}
func TestTaskRunner_SimpleRun(t *testing.T) {
ctestutil.ExecCompatible(t)
_, tr := testTaskRunner(false)
go tr.Run()
defer tr.Destroy()
defer tr.ctx.AllocDir.Destroy()
select {
case <-tr.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
if len(tr.state.Events) != 2 {
t.Fatalf("should have 2 updates: %#v", tr.state.Events)
}
if tr.state.State != structs.TaskStateDead {
t.Fatalf("TaskState %v; want %v", tr.state.State, structs.TaskStateDead)
}
if tr.state.Events[0].Type != structs.TaskStarted {
t.Fatalf("First Event was %v; want %v", tr.state.Events[0].Type, structs.TaskStarted)
}
if tr.state.Events[1].Type != structs.TaskTerminated {
t.Fatalf("First Event was %v; want %v", tr.state.Events[1].Type, structs.TaskTerminated)
}
}
func TestDockerDriver_Start_Wait(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "redis-demo",
Config: map[string]interface{}{
"image": "redis",
"command": "redis-server",
"args": []string{"-v"},
},
Resources: &structs.Resources{
MemoryMB: 256,
CPU: 512,
},
}
_, handle, cleanup := dockerSetup(t, task)
defer cleanup()
// Update should be a no-op
err := handle.Update(task)
if err != nil {
t.Fatalf("err: %v", err)
}
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}
func TestDockerDriver_Start_Kill_Wait(t *testing.T) {
t.Parallel()
task := &structs.Task{
Name: "redis-demo",
Config: map[string]interface{}{
"image": "redis",
"command": "/bin/sleep",
"args": []string{"10"},
},
Resources: basicResources,
}
_, handle, cleanup := dockerSetup(t, task)
defer cleanup()
go func() {
time.Sleep(100 * time.Millisecond)
err := handle.Kill()
if err != nil {
t.Fatalf("err: %v", err)
}
}()
select {
case res := <-handle.WaitCh():
if res.Successful() {
t.Fatalf("should err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
}
}
func TestJavaDriver_Start_Wait(t *testing.T) {
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
driverCtx, execCtx := testDriverContexts(task)
d := NewJavaDriver(driverCtx)
// Copy the test jar into the task's directory
dst, _ := execCtx.AllocDir.TaskDirs[task.Name]
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
// Task should terminate quickly
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
// expect the timeout b/c it's a long lived process
break
}
// Get the stdout of the process and assrt that it's not empty
stdout := filepath.Join(execCtx.AllocDir.LogDir(), "demo-app.stdout.0")
fInfo, err := os.Stat(stdout)
if err != nil {
t.Fatalf("failed to get stdout of process: %v", err)
}
if fInfo.Size() == 0 {
t.Fatalf("stdout of process is empty")
}
// need to kill long lived process
err = handle.Kill()
if err != nil {
t.Fatalf("Error: %s", err)
}
}
func TestExecutor_Start_Kill(t *testing.T) {
execCmd := ExecCommand{Cmd: "/bin/sleep", Args: []string{"10 && hello world"}}
ctx := testExecutorContext(t)
defer ctx.AllocDir.Destroy()
executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
ps, err := executor.LaunchCmd(&execCmd, ctx)
if err != nil {
t.Fatalf("error in launching command: %v", err)
}
if ps.Pid == 0 {
t.Fatalf("expected process to start and have non zero pid")
}
ps, err = executor.Wait()
if err != nil {
t.Fatalf("error in waiting for command: %v", err)
}
file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0")
time.Sleep(time.Duration(tu.TestMultiplier()*2) * time.Second)
output, err := ioutil.ReadFile(file)
if err != nil {
t.Fatalf("Couldn't read file %v", file)
}
expected := ""
act := strings.TrimSpace(string(output))
if act != expected {
t.Fatalf("Command output incorrectly: want %v; got %v", expected, act)
}
}
func TestDockerDriver_VolumesEnabled(t *testing.T) {
cfg := testConfig()
cfg.Options = map[string]string{dockerVolumesConfigOption: "true"}
task, driver, execCtx, hostpath, cleanup := setupDockerVolumes(t, cfg)
defer cleanup()
handle, err := driver.Start(execCtx, task)
if err != nil {
t.Fatalf("Failed to start docker driver: %v", err)
}
defer handle.Kill()
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("unexpected err: %v", res)
}
case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
}
if _, err := ioutil.ReadFile(hostpath); err != nil {
t.Fatalf("unexpected error reading %s: %v", hostpath, err)
}
}
func Executor_Start_Kill(t *testing.T, command buildExecCommand) {
task, alloc := mockAllocDir(t)
defer alloc.Destroy()
taskDir, ok := alloc.TaskDirs[task]
if !ok {
log.Panicf("No task directory found for task %v", task)
}
filePath := filepath.Join(taskDir, "output")
e := command(testtask.Path(), "sleep", "1s", "write", "failure", filePath)
if err := e.Limit(constraint); err != nil {
log.Panicf("Limit() failed: %v", err)
}
if err := e.ConfigureTaskDir(task, alloc); err != nil {
log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
}
if err := e.Start(); err != nil {
log.Panicf("Start() failed: %v", err)
}
if err := e.Shutdown(); err != nil {
log.Panicf("Shutdown() failed: %v", err)
}
time.Sleep(time.Duration(testutil.TestMultiplier()*2) * time.Second)
// Check that the file doesn't exist.
if _, err := os.Stat(filePath); err == nil {
log.Panicf("Stat(%v) should have failed: task not killed", filePath)
}
}
func TestTaskTemplateManager_Unblock_Consul(t *testing.T) {
// Make a template that will render based on a key in Consul
key := "foo"
content := "barbaz"
embedded := fmt.Sprintf(`{{key "%s"}}`, key)
file := "my.tmpl"
template := &structs.Template{
EmbeddedTmpl: embedded,
DestPath: file,
ChangeMode: structs.TemplateChangeModeNoop,
}
// Drop the retry rate
testRetryRate = 10 * time.Millisecond
harness := newTestHarness(t, []*structs.Template{template}, true, false)
harness.start(t)
defer harness.stop()
// Ensure no unblock
select {
case <-harness.mockHooks.UnblockCh:
t.Fatalf("Task unblock should have not have been called")
case <-time.After(time.Duration(1*testutil.TestMultiplier()) * time.Second):
}
// Write the key to Consul
harness.consul.SetKV(key, []byte(content))
// Wait for the unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Task unblock should have been called")
}
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
if s := string(raw); s != content {
t.Fatalf("Unexpected template data; got %q, want %q", s, content)
}
}
func TestRktDriver_Start_Wait_AllocDir(t *testing.T) {
ctestutils.RktCompatible(t)
exp := []byte{'w', 'i', 'n'}
file := "output.txt"
task := &structs.Task{
Name: "alpine",
Config: map[string]interface{}{
"image": "docker://alpine",
"command": "/bin/sh",
"args": []string{
"-c",
fmt.Sprintf(`echo -n %s > ${%s}/%s`, string(exp), env.AllocDir, file),
},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewRktDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
defer handle.Kill()
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
// Check that data was written to the shared alloc directory.
outputFile := filepath.Join(execCtx.AllocDir.SharedDir, file)
act, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Command output is %v; expected %v", act, exp)
}
}
func TestJavaDriver_Start_Wait(t *testing.T) {
t.Parallel()
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]interface{}{
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
"checksum": "sha256:58d6e8130308d32e197c5108edd4f56ddf1417408f743097c2e662df0f0b17c8",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
// Task should terminate quickly
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
// expect the timeout b/c it's a long lived process
break
}
// Get the stdout of the process and assrt that it's not empty
stdout := filepath.Join(execCtx.AllocDir.LogDir(), "demo-app.stdout.0")
fInfo, err := os.Stat(stdout)
if err != nil {
t.Fatalf("failed to get stdout of process: %v", err)
}
if fInfo.Size() == 0 {
t.Fatalf("stdout of process is empty")
}
// need to kill long lived process
err = handle.Kill()
if err != nil {
t.Fatalf("Error: %s", err)
}
}
func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
t.Parallel()
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]interface{}{
"jar_path": "demoapp.jar",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
// Copy the test jar into the task's directory
dst, _ := execCtx.AllocDir.TaskDirs[task.Name]
copyFile("./test-resources/java/demoapp.jar", filepath.Join(dst, "demoapp.jar"), t)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
go func() {
time.Sleep(100 * time.Millisecond)
err := handle.Kill()
if err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
// Need to kill long lived process
if err = handle.Kill(); err != nil {
t.Fatalf("Error: %s", err)
}
}
}
func TestTaskRunner_Destroy(t *testing.T) {
ctestutil.ExecCompatible(t)
upd, tr := testTaskRunner(true)
tr.MarkReceived()
defer tr.ctx.AllocDir.Destroy()
// Change command to ensure we run for a bit
tr.task.Config["command"] = "/bin/sleep"
tr.task.Config["args"] = []string{"1000"}
go tr.Run()
testutil.WaitForResult(func() (bool, error) {
if l := len(upd.events); l != 2 {
return false, fmt.Errorf("Expect two events; got %v", l)
}
if upd.events[0].Type != structs.TaskReceived {
return false, fmt.Errorf("First Event was %v; want %v", upd.events[0].Type, structs.TaskReceived)
}
if upd.events[1].Type != structs.TaskStarted {
return false, fmt.Errorf("Second Event was %v; want %v", upd.events[1].Type, structs.TaskStarted)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Make sure we are collecting afew stats
time.Sleep(2 * time.Second)
stats := tr.StatsReporter().ResourceUsage()
if len(stats) == 0 {
t.Fatalf("expected task runner to have some stats")
}
// Begin the tear down
tr.Destroy()
select {
case <-tr.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
if len(upd.events) != 3 {
t.Fatalf("should have 3 updates: %#v", upd.events)
}
if upd.state != structs.TaskStateDead {
t.Fatalf("TaskState %v; want %v", upd.state, structs.TaskStateDead)
}
if upd.events[2].Type != structs.TaskKilled {
t.Fatalf("Third Event was %v; want %v", upd.events[2].Type, structs.TaskKilled)
}
}
// TestAllocRuner_RetryArtifact ensures that if one task in a task group is
// retrying fetching an artifact, other tasks in the group should be able
// to proceed.
func TestAllocRunner_RetryArtifact(t *testing.T) {
ctestutil.ExecCompatible(t)
alloc := mock.Alloc()
alloc.Job.Type = structs.JobTypeBatch
alloc.Job.TaskGroups[0].RestartPolicy.Mode = structs.RestartPolicyModeFail
alloc.Job.TaskGroups[0].RestartPolicy.Attempts = 1
alloc.Job.TaskGroups[0].RestartPolicy.Delay = time.Duration(4*testutil.TestMultiplier()) * time.Second
task := alloc.Job.TaskGroups[0].Tasks[0]
task.Driver = "mock_driver"
task.Config = map[string]interface{}{
"exit_code": "0",
"run_for": "1s",
}
// Create a new task with a bad artifact
badtask := alloc.Job.TaskGroups[0].Tasks[0].Copy()
badtask.Name = "bad"
badtask.Artifacts = []*structs.TaskArtifact{
{GetterSource: "http://127.1.1.111:12315/foo/bar/baz"},
}
alloc.Job.TaskGroups[0].Tasks = append(alloc.Job.TaskGroups[0].Tasks, badtask)
upd, ar := testAllocRunnerFromAlloc(alloc, true)
go ar.Run()
defer ar.Destroy()
testutil.WaitForResult(func() (bool, error) {
if upd.Count < 6 {
return false, fmt.Errorf("Not enough updates")
}
last := upd.Allocs[upd.Count-1]
// web task should have completed successfully while bad task
// retries artififact fetching
webstate := last.TaskStates["web"]
if webstate.State != structs.TaskStateDead {
return false, fmt.Errorf("expected web to be dead but found %q", last.TaskStates["web"].State)
}
if !webstate.Successful() {
return false, fmt.Errorf("expected web to have exited successfully")
}
// bad task should have failed
badstate := last.TaskStates["bad"]
if badstate.State != structs.TaskStateDead {
return false, fmt.Errorf("expected bad to be dead but found %q", badstate.State)
}
if !badstate.Failed {
return false, fmt.Errorf("expected bad to have failed: %#v", badstate.Events)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
func TestRktDriver_Start_Wait(t *testing.T) {
if os.Getenv("NOMAD_TEST_RKT") == "" {
t.Skip("skipping rkt tests")
}
ctestutils.RktCompatible(t)
task := &structs.Task{
Name: "etcd",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": []string{"--version"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
MemoryMB: 128,
CPU: 100,
},
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewRktDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
defer handle.Kill()
// Update should be a no-op
err = handle.Update(task)
if err != nil {
t.Fatalf("err: %v", err)
}
// Signal should be an error
if err = handle.Signal(syscall.SIGTERM); err == nil {
t.Fatalf("err: %v", err)
}
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}
func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
t.Parallel()
if !javaLocated() {
t.Skip("Java not found; skipping")
}
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "demo-app",
Config: map[string]interface{}{
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
go func() {
time.Sleep(100 * time.Millisecond)
err := handle.Kill()
if err != nil {
t.Fatalf("err: %v", err)
}
}()
// Task should terminate quickly
select {
case res := <-handle.WaitCh():
if res.Successful() {
t.Fatal("should err")
}
case <-time.After(time.Duration(testutil.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
}
// need to kill long lived process
err = handle.Kill()
if err != nil {
t.Fatalf("Error: %s", err)
}
}
func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) {
t.Parallel()
exp := []byte{'w', 'i', 'n'}
file := "output.txt"
outPath := fmt.Sprintf(`${%s}/%s`, env.AllocDir, file)
task := &structs.Task{
Name: "sleep",
Config: map[string]interface{}{
"command": testtask.Path(),
"args": []string{
"sleep", "1s",
"write", string(exp), outPath,
},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewRawExecDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
// Task should terminate quickly
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
// Check that data was written to the shared alloc directory.
outputFile := filepath.Join(execCtx.AllocDir.SharedDir, file)
act, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatalf("Couldn't read expected output: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Command outputted %v; want %v", act, exp)
}
}
func TestTaskRunner_KillTask(t *testing.T) {
alloc := mock.Alloc()
task := alloc.Job.TaskGroups[0].Tasks[0]
task.Driver = "mock_driver"
task.Config = map[string]interface{}{
"exit_code": "0",
"run_for": "10s",
}
upd, tr := testTaskRunnerFromAlloc(false, alloc)
tr.MarkReceived()
go tr.Run()
defer tr.Destroy(structs.NewTaskEvent(structs.TaskKilled))
defer tr.ctx.AllocDir.Destroy()
go func() {
time.Sleep(100 * time.Millisecond)
tr.Kill("test", "kill", true)
}()
select {
case <-tr.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
t.Fatalf("timeout")
}
if len(upd.events) != 4 {
t.Fatalf("should have 4 updates: %#v", upd.events)
}
if upd.state != structs.TaskStateDead {
t.Fatalf("TaskState %v; want %v", upd.state, structs.TaskStateDead)
}
if !upd.failed {
t.Fatalf("TaskState should be failed: %+v", upd)
}
if upd.events[0].Type != structs.TaskReceived {
t.Fatalf("First Event was %v; want %v", upd.events[0].Type, structs.TaskReceived)
}
if upd.events[1].Type != structs.TaskStarted {
t.Fatalf("Second Event was %v; want %v", upd.events[1].Type, structs.TaskStarted)
}
if upd.events[2].Type != structs.TaskKilling {
t.Fatalf("Third Event was %v; want %v", upd.events[2].Type, structs.TaskKilling)
}
if upd.events[3].Type != structs.TaskKilled {
t.Fatalf("Fourth Event was %v; want %v", upd.events[3].Type, structs.TaskKilled)
}
}
func TestRktDriver_Start_Wait_Logs(t *testing.T) {
ctestutils.RktCompatible(t)
task := &structs.Task{
Name: "etcd",
Config: map[string]interface{}{
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": []string{"--version"},
},
Resources: &structs.Resources{
MemoryMB: 256,
CPU: 512,
},
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewRktDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
defer handle.Kill()
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
taskDir, ok := execCtx.AllocDir.TaskDirs[task.Name]
if !ok {
t.Fatalf("Could not find task directory for task: %v", task)
}
stdout := filepath.Join(taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", task.Name))
data, err := ioutil.ReadFile(stdout)
if err != nil {
t.Fatalf("Failed to read tasks stdout: %v", err)
}
if len(data) == 0 {
t.Fatal("Task's stdout is empty")
}
}
func TestDockerDriver_VolumesDisabled(t *testing.T) {
cfg := testConfig()
cfg.Options = map[string]string{
dockerVolumesConfigOption: "false",
"docker.cleanup.image": "false",
}
{
tmpvol, err := ioutil.TempDir("", "nomadtest_docker_volumesdisabled")
if err != nil {
t.Fatalf("error creating temporary dir: %v", err)
}
task, driver, execCtx, _, cleanup := setupDockerVolumes(t, cfg, tmpvol)
defer cleanup()
if _, err := driver.Start(execCtx, task); err == nil {
t.Fatalf("Started driver successfully when volumes should have been disabled.")
}
}
// Relative paths should still be allowed
{
task, driver, execCtx, fn, cleanup := setupDockerVolumes(t, cfg, ".")
defer cleanup()
handle, err := driver.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
defer handle.Kill()
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("unexpected err: %v", res)
}
case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second):
t.Fatalf("timeout")
}
taskDir, ok := execCtx.AllocDir.TaskDirs[task.Name]
if !ok {
t.Fatalf("Failed to get task dir")
}
if _, err := ioutil.ReadFile(filepath.Join(taskDir, fn)); err != nil {
t.Fatalf("unexpected error reading %s: %v", fn, err)
}
}
}
func TestTaskTemplateManager_HostPath(t *testing.T) {
// Make a template that will render immediately and write it to a tmp file
f, err := ioutil.TempFile("", "")
if err != nil {
t.Fatalf("Bad: %v", err)
}
defer f.Close()
defer os.Remove(f.Name())
content := "hello, world!"
if _, err := io.WriteString(f, content); err != nil {
t.Fatalf("Bad: %v", err)
}
file := "my.tmpl"
template := &structs.Template{
SourcePath: f.Name(),
DestPath: file,
ChangeMode: structs.TemplateChangeModeNoop,
}
harness := newTestHarness(t, []*structs.Template{template}, false, false)
harness.start(t)
defer harness.stop()
// Wait for the unblock
select {
case <-harness.mockHooks.UnblockCh:
case <-time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second):
t.Fatalf("Task unblock should have been called")
}
// Check the file is there
path := filepath.Join(harness.taskDir, file)
raw, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read rendered template from %q: %v", path, err)
}
if s := string(raw); s != content {
t.Fatalf("Unexpected template data; got %q, want %q", s, content)
}
// Change the config to disallow host sources
harness = newTestHarness(t, []*structs.Template{template}, false, false)
harness.config.Options = map[string]string{
hostSrcOption: "false",
}
if err := harness.startWithErr(); err == nil || !strings.Contains(err.Error(), "absolute") {
t.Fatalf("Expected absolute template path disallowed: %v", err)
}
}
func TestAllocRunner_SaveRestoreState(t *testing.T) {
ctestutil.ExecCompatible(t)
upd, ar := testAllocRunner(false)
// Ensure task takes some time
task := ar.alloc.Job.TaskGroups[0].Tasks[0]
task.Config["command"] = "/bin/sleep"
task.Config["args"] = []string{"10"}
go ar.Run()
defer ar.Destroy()
// Snapshot state
testutil.WaitForResult(func() (bool, error) {
return len(ar.tasks) == 1, nil
}, func(err error) {
t.Fatalf("task never started: %v", err)
})
err := ar.SaveState()
if err != nil {
t.Fatalf("err: %v", err)
}
// Create a new alloc runner
consulClient, err := NewConsulService(&consulServiceConfig{ar.logger, "127.0.0.1:8500", "", "", false, false, &structs.Node{}})
ar2 := NewAllocRunner(ar.logger, ar.config, upd.Update,
&structs.Allocation{ID: ar.alloc.ID}, consulClient)
err = ar2.RestoreState()
if err != nil {
t.Fatalf("err: %v", err)
}
go ar2.Run()
// Destroy and wait
ar2.Destroy()
start := time.Now()
testutil.WaitForResult(func() (bool, error) {
if upd.Count == 0 {
return false, nil
}
last := upd.Allocs[upd.Count-1]
return last.ClientStatus != structs.AllocClientStatusPending, nil
}, func(err error) {
t.Fatalf("err: %v %#v %#v", err, upd.Allocs[0], ar.alloc.TaskStates)
})
if time.Since(start) > time.Duration(testutil.TestMultiplier()*15)*time.Second {
t.Fatalf("took too long to terminate")
}
}
func TestRawExecDriver_Start_Artifact_basic(t *testing.T) {
t.Parallel()
path := testtask.Path()
ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir(path))))
defer ts.Close()
file := filepath.Base(path)
task := &structs.Task{
Name: "sleep",
Config: map[string]interface{}{
"artifact_source": fmt.Sprintf("%s/%s", ts.URL, file),
"command": file,
"args": []string{"sleep", "1s"},
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: basicResources,
}
testtask.SetTaskEnv(task)
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewRawExecDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
// Attempt to open
handle2, err := d.Open(execCtx, handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle")
}
// Task should terminate quickly
select {
case <-handle2.WaitCh():
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}