func TestAllocDir_Move(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
// Create two alloc dirs
d1 := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d1.Destroy()
d2 := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d2.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d1.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
if err := d2.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
dataDir := filepath.Join(d1.SharedDir, "data")
taskDir := d1.TaskDirs[t1.Name]
taskLocal := filepath.Join(taskDir, "local")
// Write a file to the shared dir.
exp := []byte{'f', 'o', 'o'}
file := "bar"
if err := ioutil.WriteFile(filepath.Join(dataDir, file), exp, 0777); err != nil {
t.Fatalf("Couldn't write file to shared directory: %v", err)
}
// Write a file to the task local
exp = []byte{'b', 'a', 'r'}
file1 := "lol"
if err := ioutil.WriteFile(filepath.Join(taskLocal, file1), exp, 0777); err != nil {
t.Fatalf("couldn't write to task local directory: %v", err)
}
// Move the d1 allocdir to d2
if err := d2.Move(d1, []*structs.Task{t1, t2}); err != nil {
t.Fatalf("err: %v", err)
}
// Ensure the files in d1 are present in d2
fi, err := os.Stat(filepath.Join(d2.SharedDir, "data", "bar"))
if err != nil || fi == nil {
t.Fatalf("data dir was not moved")
}
fi, err = os.Stat(filepath.Join(d2.TaskDirs[t1.Name], "local", "lol"))
if err != nil || fi == nil {
t.Fatalf("task local dir was not moved")
}
}
// Test that given a set of tasks, each task gets a directory and that directory
// has the shared alloc dir inside of it.
func TestAllocDir_BuildAlloc(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Check that the AllocDir and each of the task directories exist.
if _, err := os.Stat(d.AllocDir); os.IsNotExist(err) {
t.Fatalf("Build(%v) didn't create AllocDir %v", tasks, d.AllocDir)
}
for _, task := range tasks {
tDir, ok := d.TaskDirs[task.Name]
if !ok {
t.Fatalf("Task directory not found for %v", task.Name)
}
if _, err := os.Stat(tDir); os.IsNotExist(err) {
t.Fatalf("Build(%v) didn't create TaskDir %v", tasks, tDir)
}
if _, err := os.Stat(filepath.Join(tDir, TaskSecrets)); os.IsNotExist(err) {
t.Fatalf("Build(%v) didn't create secret dir %v", tasks)
}
}
}
// The fingerprinter test should always pass, even if Java is not installed.
func TestJavaDriver_Fingerprint(t *testing.T) {
ctestutils.JavaCompatible(t)
task := &structs.Task{
Name: "foo",
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
node := &structs.Node{
Attributes: map[string]string{
"unique.cgroup.mountpoint": "/sys/fs/cgroups",
},
}
apply, err := d.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if apply != javaLocated() {
t.Fatalf("Fingerprinter should detect Java when it is installed")
}
if node.Attributes["driver.java"] != "1" {
if v, ok := osJavaDriverSupport[runtime.GOOS]; v && ok {
t.Fatalf("missing java driver")
} else {
t.Skipf("missing java driver, no OS support")
}
}
for _, key := range []string{"driver.java.version", "driver.java.runtime", "driver.java.vm"} {
if node.Attributes[key] == "" {
t.Fatalf("missing driver key (%s)", key)
}
}
}
// The fingerprinter test should always pass, even if QEMU is not installed.
func TestQemuDriver_Fingerprint(t *testing.T) {
ctestutils.QemuCompatible(t)
task := &structs.Task{
Name: "foo",
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewQemuDriver(driverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
}
apply, err := d.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if !apply {
t.Fatalf("should apply")
}
if node.Attributes["driver.qemu"] == "" {
t.Fatalf("Missing Qemu driver")
}
if node.Attributes["driver.qemu.version"] == "" {
t.Fatalf("Missing Qemu driver version")
}
}
func TestLxcDriver_Fingerprint(t *testing.T) {
if !lxcPresent(t) {
t.Skip("lxc not present")
}
task := &structs.Task{
Name: "foo",
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewLxcDriver(driverCtx)
node := &structs.Node{
Attributes: map[string]string{},
}
apply, err := d.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if !apply {
t.Fatalf("should apply by default")
}
apply, err = d.Fingerprint(&config.Config{Options: map[string]string{lxcConfigOption: "0"}}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if apply {
t.Fatalf("should not apply with config")
}
if node.Attributes["driver.lxc"] == "" {
t.Fatalf("missing driver")
}
}
func TestLxcDriver_Open_Wait(t *testing.T) {
if !lxcPresent(t) {
t.Skip("lxc not present")
}
task := &structs.Task{
Name: "foo",
Config: map[string]interface{}{
"template": "/usr/share/lxc/templates/lxc-busybox",
},
KillTimeout: 10 * time.Second,
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewLxcDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
// Destroy the container after the test
if lh, ok := handle.(*lxcDriverHandle); ok {
defer func() {
lh.container.Stop()
lh.container.Destroy()
}()
}
handle2, err := d.Open(execCtx, handle.ID())
if err != nil {
t.Fatalf("err: %v", err)
}
if handle2 == nil {
t.Fatalf("missing handle on open")
}
lxcHandle, _ := handle2.(*lxcDriverHandle)
testutil.WaitForResult(func() (bool, error) {
state := lxcHandle.container.State()
if state == lxc.RUNNING {
return true, nil
}
return false, fmt.Errorf("container in state: %v", state)
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Desroy the container
if err := handle2.Kill(); err != nil {
t.Fatalf("err: %v", err)
}
}
func TestExecDriver_Fingerprint(t *testing.T) {
ctestutils.ExecCompatible(t)
task := &structs.Task{
Name: "foo",
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewExecDriver(driverCtx)
node := &structs.Node{
Attributes: map[string]string{
"unique.cgroup.mountpoint": "/sys/fs/cgroup",
},
}
apply, err := d.Fingerprint(&config.Config{}, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if !apply {
t.Fatalf("should apply")
}
if node.Attributes["driver.exec"] == "" {
t.Fatalf("missing driver")
}
}
func TestAllocDir_EmbedDirs(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Create a fake host directory, with a file, and a subfolder that contains
// a file.
host, err := ioutil.TempDir("", "AllocDirHost")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(host)
subDirName := "subdir"
subDir := filepath.Join(host, subDirName)
if err := os.MkdirAll(subDir, 0777); err != nil {
t.Fatalf("Failed to make subdir %v: %v", subDir, err)
}
file := "foo"
subFile := "bar"
if err := ioutil.WriteFile(filepath.Join(host, file), []byte{'a'}, 0777); err != nil {
t.Fatalf("Coudn't create file in host dir %v: %v", host, err)
}
if err := ioutil.WriteFile(filepath.Join(subDir, subFile), []byte{'a'}, 0777); err != nil {
t.Fatalf("Coudn't create file in host subdir %v: %v", subDir, err)
}
// Create mapping from host dir to task dir.
task := tasks[0].Name
taskDest := "bin/test/"
mapping := map[string]string{host: taskDest}
if err := d.Embed(task, mapping); err != nil {
t.Fatalf("Embed(%v, %v) failed: %v", task, mapping, err)
}
// Check that the embedding was done properly.
taskDir, ok := d.TaskDirs[task]
if !ok {
t.Fatalf("Task directory not found for %v", task)
}
exp := []string{filepath.Join(taskDir, taskDest, file), filepath.Join(taskDir, taskDest, subDirName, subFile)}
for _, e := range exp {
if _, err := os.Stat(e); os.IsNotExist(err) {
t.Fatalf("File %v not embeded: %v", e, err)
}
}
}
func TestAllocDir_Snapshot(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
dataDir := filepath.Join(d.SharedDir, "data")
taskDir := d.TaskDirs[t1.Name]
taskLocal := filepath.Join(taskDir, "local")
// Write a file to the shared dir.
exp := []byte{'f', 'o', 'o'}
file := "bar"
if err := ioutil.WriteFile(filepath.Join(dataDir, file), exp, 0777); err != nil {
t.Fatalf("Couldn't write file to shared directory: %v", err)
}
// Write a file to the task local
exp = []byte{'b', 'a', 'r'}
file1 := "lol"
if err := ioutil.WriteFile(filepath.Join(taskLocal, file1), exp, 0777); err != nil {
t.Fatalf("couldn't write to task local directory: %v", err)
}
var b bytes.Buffer
if err := d.Snapshot(&b); err != nil {
t.Fatalf("err: %v", err)
}
tr := tar.NewReader(&b)
var files []string
for {
hdr, err := tr.Next()
if err != nil && err != io.EOF {
t.Fatalf("err: %v", err)
}
if err == io.EOF {
break
}
if hdr.Typeflag == tar.TypeReg {
files = append(files, hdr.FileInfo().Name())
}
}
if len(files) != 2 {
t.Fatalf("bad files: %#v", files)
}
}
// tempAllocDir returns a new alloc dir that is rooted in a temp dir. The caller
// should destroy the temp dir.
func tempAllocDir(t *testing.T) *allocdir.AllocDir {
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("TempDir() failed: %v", err)
}
if err := os.Chmod(dir, 0777); err != nil {
t.Fatalf("failed to chmod dir: %v", err)
}
return allocdir.NewAllocDir(dir, structs.DefaultResources().DiskMB)
}
func TestAllocDir_LogDir(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
expected := filepath.Join(d.AllocDir, SharedAllocName, LogDirName)
if d.LogDir() != expected {
t.Fatalf("expected: %v, got: %v", expected, d.LogDir())
}
}
func TestAllocDir_MountSharedAlloc(t *testing.T) {
testutil.MountCompatible(t)
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Write a file to the shared dir.
exp := []byte{'f', 'o', 'o'}
file := "bar"
if err := ioutil.WriteFile(filepath.Join(d.SharedDir, file), exp, 0777); err != nil {
t.Fatalf("Couldn't write file to shared directory: %v", err)
}
for _, task := range tasks {
// Mount and then check that the file exists in the task directory.
if err := d.MountSharedDir(task.Name); err != nil {
if v, ok := osMountSharedDirSupport[runtime.GOOS]; v && ok {
t.Fatalf("MountSharedDir(%v) failed: %v", task.Name, err)
} else {
t.Skipf("MountShareDir(%v) failed, no OS support")
}
}
taskDir, ok := d.TaskDirs[task.Name]
if !ok {
t.Fatalf("Task directory not found for %v", task.Name)
}
taskFile := filepath.Join(taskDir, SharedAllocName, file)
act, err := ioutil.ReadFile(taskFile)
if err != nil {
t.Fatalf("Failed to read shared alloc file from task dir: %v", err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("Incorrect data read from task dir: want %v; got %v", exp, act)
}
}
}
func TestAllocDir_EscapeChecking(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
// Check that issuing calls that escape the alloc dir returns errors
// List
if _, err := d.List(".."); err == nil || !strings.Contains(err.Error(), "escapes") {
t.Fatalf("List of escaping path didn't error: %v", err)
}
// Stat
if _, err := d.Stat("../foo"); err == nil || !strings.Contains(err.Error(), "escapes") {
t.Fatalf("Stat of escaping path didn't error: %v", err)
}
// ReadAt
if _, err := d.ReadAt("../foo", 0); err == nil || !strings.Contains(err.Error(), "escapes") {
t.Fatalf("ReadAt of escaping path didn't error: %v", err)
}
// BlockUntilExists
tomb := tomb.Tomb{}
if _, err := d.BlockUntilExists("../foo", &tomb); err == nil || !strings.Contains(err.Error(), "escapes") {
t.Fatalf("BlockUntilExists of escaping path didn't error: %v", err)
}
// ChangeEvents
if _, err := d.ChangeEvents("../foo", 0, &tomb); err == nil || !strings.Contains(err.Error(), "escapes") {
t.Fatalf("ChangeEvents of escaping path didn't error: %v", err)
}
}
func TestAllocDir_EmbedNonExistent(t *testing.T) {
tmp, err := ioutil.TempDir("", "AllocDir")
if err != nil {
t.Fatalf("Couldn't create temp dir: %v", err)
}
defer os.RemoveAll(tmp)
d := NewAllocDir(tmp, structs.DefaultResources().DiskMB)
defer d.Destroy()
tasks := []*structs.Task{t1, t2}
if err := d.Build(tasks); err != nil {
t.Fatalf("Build(%v) failed: %v", tasks, err)
}
fakeDir := "/foobarbaz"
task := tasks[0].Name
mapping := map[string]string{fakeDir: fakeDir}
if err := d.Embed(task, mapping); err != nil {
t.Fatalf("Embed(%v, %v) should should skip %v since it does not exist", task, mapping, fakeDir)
}
}
func TestRawExecDriver_Fingerprint(t *testing.T) {
task := &structs.Task{
Name: "foo",
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewRawExecDriver(driverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
}
// Disable raw exec.
cfg := &config.Config{Options: map[string]string{rawExecConfigOption: "false"}}
apply, err := d.Fingerprint(cfg, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if apply {
t.Fatalf("should not apply")
}
if node.Attributes["driver.raw_exec"] != "" {
t.Fatalf("driver incorrectly enabled")
}
// Enable raw exec.
cfg.Options[rawExecConfigOption] = "true"
apply, err = d.Fingerprint(cfg, node)
if err != nil {
t.Fatalf("err: %v", err)
}
if !apply {
t.Fatalf("should apply")
}
if node.Attributes["driver.raw_exec"] != "1" {
t.Fatalf("driver not enabled")
}
}
func parseResources(result *structs.Resources, list *ast.ObjectList) error {
list = list.Elem()
if len(list.Items) == 0 {
return nil
}
if len(list.Items) > 1 {
return fmt.Errorf("only one 'resource' block allowed per task")
}
// Get our resource object
o := list.Items[0]
// We need this later
var listVal *ast.ObjectList
if ot, ok := o.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return fmt.Errorf("resource: should be an object")
}
// Check for invalid keys
valid := []string{
"cpu",
"iops",
"memory",
"network",
}
if err := checkHCLKeys(listVal, valid); err != nil {
return multierror.Prefix(err, "resources ->")
}
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Val); err != nil {
return err
}
delete(m, "network")
if err := mapstructure.WeakDecode(m, result); err != nil {
return err
}
// Parse the network resources
if o := listVal.Filter("network"); len(o.Items) > 0 {
if len(o.Items) > 1 {
return fmt.Errorf("only one 'network' resource allowed")
}
// Check for invalid keys
valid := []string{
"mbits",
"port",
}
if err := checkHCLKeys(o.Items[0].Val, valid); err != nil {
return multierror.Prefix(err, "resources, network ->")
}
var r structs.NetworkResource
var m map[string]interface{}
if err := hcl.DecodeObject(&m, o.Items[0].Val); err != nil {
return err
}
if err := mapstructure.WeakDecode(m, &r); err != nil {
return err
}
var networkObj *ast.ObjectList
if ot, ok := o.Items[0].Val.(*ast.ObjectType); ok {
networkObj = ot.List
} else {
return fmt.Errorf("resource: should be an object")
}
if err := parsePorts(networkObj, &r); err != nil {
return multierror.Prefix(err, "resources, network, ports ->")
}
result.Networks = []*structs.NetworkResource{&r}
}
// Combine the parsed resources with a default resource block.
min := structs.DefaultResources()
min.Merge(result)
*result = *min
return nil
}
func TestLxcDriver_Start_Wait(t *testing.T) {
if !lxcPresent(t) {
t.Skip("lxc not present")
}
task := &structs.Task{
Name: "foo",
Config: map[string]interface{}{
"template": "/usr/share/lxc/templates/lxc-busybox",
},
KillTimeout: 10 * time.Second,
Resources: structs.DefaultResources(),
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
d := NewLxcDriver(driverCtx)
handle, err := d.Start(execCtx, task)
if err != nil {
t.Fatalf("err: %v", err)
}
if handle == nil {
t.Fatalf("missing handle")
}
lxcHandle, _ := handle.(*lxcDriverHandle)
// Destroy the container after the test
defer func() {
lxcHandle.container.Stop()
lxcHandle.container.Destroy()
}()
testutil.WaitForResult(func() (bool, error) {
state := lxcHandle.container.State()
if state == lxc.RUNNING {
return true, nil
}
return false, fmt.Errorf("container in state: %v", state)
}, func(err error) {
t.Fatalf("err: %v", err)
})
// Look for mounted directories in their proper location
containerName := fmt.Sprintf("%s-%s", task.Name, execCtx.AllocID)
for _, mnt := range []string{"alloc", "local", "secret"} {
fullpath := filepath.Join(lxcHandle.lxcPath, containerName, "rootfs", mnt)
stat, err := os.Stat(fullpath)
if err != nil {
t.Fatalf("err %v", err)
}
if !stat.IsDir() {
t.Fatalf("expected %q to be a dir", fullpath)
}
}
// Desroy the container
if err := handle.Kill(); err != nil {
t.Fatalf("err: %v", err)
}
select {
case res := <-handle.WaitCh():
if !res.Successful() {
t.Fatalf("err: %v", res)
}
case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
t.Fatalf("timeout")
}
}