func TestEncrypt(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-node", "test1",
"-server",
"-tag", "role=test",
"-encrypt", "kPpdjphiipNSsjd4QHWbkA==",
"-log-level", logLevel,
}
go a.Run(args)
time.Sleep(2 * time.Second)
assert.True(t, a.serf.EncryptionEnabled())
shutdownCh <- struct{}{}
}
func testConfig() *Config {
config := DefaultConfig()
config.Init()
config.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
// Set probe intervals that are aggressive for finding bad nodes
config.MemberlistConfig.GossipInterval = 5 * time.Millisecond
config.MemberlistConfig.ProbeInterval = 50 * time.Millisecond
config.MemberlistConfig.ProbeTimeout = 25 * time.Millisecond
config.MemberlistConfig.SuspicionMult = 1
config.NodeName = fmt.Sprintf("Node %s", config.MemberlistConfig.BindAddr)
// Set a short reap interval so that it can run during the test
config.ReapInterval = 1 * time.Second
// Set a short reconnect interval so that it can run a lot during tests
config.ReconnectInterval = 100 * time.Millisecond
// Set basically zero on the reconnect/tombstone timeouts so that
// they're removed on the first ReapInterval.
config.ReconnectTimeout = 1 * time.Microsecond
config.TombstoneTimeout = 1 * time.Microsecond
return config
}
func setupAPITest(t *testing.T) (chan<- struct{}, <-chan int) {
log.Level = logrus.FatalLevel
shutdownCh := make(chan struct{})
// defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-http-addr", "127.0.0.1:8090",
"-node", "test",
"-server",
}
resultCh := make(chan int)
go func() {
resultCh <- a.Run(args)
}()
time.Sleep(10 * time.Millisecond)
return shutdownCh, resultCh
}
func TestEncrypt(t *testing.T) {
log.Level = logrus.ErrorLevel
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-node", "test1",
"-server",
"-tag", "role=test",
"-encrypt", "kPpdjphiipNSsjd4QHWbkA==",
}
go func() {
a.Run(args)
}()
time.Sleep(2 * time.Second)
if !a.serf.EncryptionEnabled() {
t.Fatal("Encryption not enabled for serf")
}
shutdownCh <- struct{}{}
}
func Test_getRPCAddr(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
a1Addr := testutil.GetBindAddr()
args := []string{
"-bind", a1Addr.String() + ":5000",
"-node", "test1",
"-server",
"-tag", "role=test",
}
go a.Run(args)
time.Sleep(2 * time.Second)
getRPCAddr := a.getRPCAddr()
exRPCAddr := a1Addr.String() + ":6868"
if exRPCAddr != getRPCAddr {
t.Fatalf("Expected address was: %s got %s", exRPCAddr, getRPCAddr)
}
shutdownCh <- struct{}{}
}
func Test_getRPCAddr(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
a1Addr := testutil.GetBindAddr()
args := []string{
"-bind", a1Addr.String() + ":5000",
"-node", "test1",
"-server",
"-tag", "role=test",
"-log-level", logLevel,
}
go a.Run(args)
time.Sleep(2 * time.Second)
getRPCAddr := a.getRPCAddr()
exRPCAddr := a1Addr.String() + ":6868"
assert.Equal(t, exRPCAddr, getRPCAddr)
shutdownCh <- struct{}{}
}
func TestCommandRun_joinFail(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
c := &Command{
ShutdownCh: shutdownCh,
Ui: new(cli.MockUi),
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-join", testutil.GetBindAddr().String(),
}
code := c.Run(args)
if code == 0 {
t.Fatal("should fail")
}
}
func TestCommandRun_advertiseAddr(t *testing.T) {
doneCh := make(chan struct{})
shutdownCh := make(chan struct{})
defer func() {
close(shutdownCh)
<-doneCh
}()
c := &Command{
ShutdownCh: shutdownCh,
Ui: new(cli.MockUi),
}
rpcAddr := getRPCAddr()
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-rpc-addr", rpcAddr,
"-advertise", "127.0.0.10:12345",
}
go func() {
code := c.Run(args)
if code != 0 {
log.Printf("bad: %d", code)
}
close(doneCh)
}()
testutil.Yield()
client, err := rpc.NewClient(rpcAddr)
if err != nil {
t.Fatalf("err: %s", err)
}
defer client.Close()
members, err := client.Members()
if err != nil {
t.Fatalf("err: %s", err)
}
if len(members) != 1 {
t.Fatalf("bad: %#v", members)
}
// Check the addr and port is as advertised!
m := members[0]
if bytes.Compare(m.Addr, []byte{127, 0, 0, 10}) != 0 {
t.Fatalf("bad: %#v", m)
}
if m.Port != 12345 {
t.Fatalf("bad: %#v", m)
}
}
func testAgent() *Agent {
config := serf.DefaultConfig()
config.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
config.NodeName = config.MemberlistConfig.BindAddr
agent := &Agent{
RPCAddr: getRPCAddr(),
SerfConfig: config,
}
return agent
}
func TestAgentConfig(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
advAddr := testutil.GetBindAddr().String()
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-advertise", advAddr,
"-log-level", logLevel,
}
resultCh := make(chan int)
go func() {
resultCh <- a.Run(args)
}()
time.Sleep(2 * time.Second)
assert.NotEqual(t, a.config.AdvertiseAddr, a.config.BindAddr)
assert.NotEmpty(t, a.config.AdvertiseAddr)
assert.Equal(t, advAddr, a.config.AdvertiseAddr)
// Send a shutdown request
shutdownCh <- struct{}{}
select {
case code := <-resultCh:
if code != 0 {
t.Fatalf("bad code: %d", code)
}
case <-time.After(100 * time.Millisecond):
t.Fatalf("timeout")
}
}
func testAgent(logOutput io.Writer) *Agent {
if logOutput == nil {
logOutput = os.Stderr
}
config := serf.DefaultConfig()
config.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
config.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
config.NodeName = config.MemberlistConfig.BindAddr
agent, err := Create(config, logOutput)
if err != nil {
panic(err)
}
return agent
}
func testAgentWithConfig(agentConfig *Config, serfConfig *serf.Config,
logOutput io.Writer) *Agent {
if logOutput == nil {
logOutput = os.Stderr
}
serfConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
serfConfig.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
serfConfig.NodeName = serfConfig.MemberlistConfig.BindAddr
agent, err := Create(agentConfig, serfConfig, logOutput)
if err != nil {
panic(err)
}
return agent
}
func testAgent(t *testing.T) *agent.Agent {
config := serf.DefaultConfig()
config.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
config.NodeName = config.MemberlistConfig.BindAddr
agent := &agent.Agent{
RPCAddr: getRPCAddr(),
SerfConfig: config,
}
if err := agent.Start(); err != nil {
t.Fatalf("err: %s", err)
}
return agent
}
func TestCommandRun_rpc(t *testing.T) {
doneCh := make(chan struct{})
shutdownCh := make(chan struct{})
defer func() {
close(shutdownCh)
<-doneCh
}()
c := &Command{
ShutdownCh: shutdownCh,
Ui: new(cli.MockUi),
}
rpcAddr := getRPCAddr()
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-rpc-addr", rpcAddr,
}
go func() {
code := c.Run(args)
if code != 0 {
log.Printf("bad: %d", code)
}
close(doneCh)
}()
testutil.Yield()
client, err := rpc.NewClient(rpcAddr)
if err != nil {
t.Fatalf("err: %s", err)
}
defer client.Close()
members, err := client.Members()
if err != nil {
t.Fatalf("err: %s", err)
}
if len(members) != 1 {
t.Fatalf("bad: %#v", members)
}
}
func testAgent(t *testing.T) *agent.Agent {
config := serf.DefaultConfig()
config.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
config.MemberlistConfig.ProbeInterval = 50 * time.Millisecond
config.MemberlistConfig.ProbeTimeout = 25 * time.Millisecond
config.MemberlistConfig.SuspicionMult = 1
config.NodeName = config.MemberlistConfig.BindAddr
agent := &agent.Agent{
RPCAddr: getRPCAddr(),
SerfConfig: config,
}
if err := agent.Start(); err != nil {
t.Fatalf("err: %s", err)
}
return agent
}
func TestAgentCommandRun(t *testing.T) {
log.Level = logrus.FatalLevel
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
}
resultCh := make(chan int)
go func() {
resultCh <- a.Run(args)
}()
time.Sleep(2 * time.Second)
// Verify it runs "forever"
select {
case <-resultCh:
t.Fatalf("ended too soon, err: %s", ui.ErrorWriter.String())
case <-time.After(50 * time.Millisecond):
}
// Send a shutdown request
shutdownCh <- struct{}{}
select {
case code := <-resultCh:
if code != 0 {
t.Fatalf("bad code: %d", code)
}
case <-time.After(50 * time.Millisecond):
t.Fatalf("timeout")
}
}
func testAgent(t *testing.T) *agent.Agent {
config := serf.DefaultConfig()
config.MemberlistConfig.BindAddr = testutil.GetBindAddr().String()
config.MemberlistConfig.ProbeInterval = 50 * time.Millisecond
config.MemberlistConfig.ProbeTimeout = 25 * time.Millisecond
config.MemberlistConfig.SuspicionMult = 1
config.NodeName = config.MemberlistConfig.BindAddr
config.Tags = map[string]string{"role": "test", "tag1": "foo", "tag2": "bar"}
agent, err := agent.Create(config, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
if err := agent.Start(); err != nil {
t.Fatalf("err: %s", err)
}
return agent
}
func TestCommandRun(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
c := &Command{
ShutdownCh: shutdownCh,
Ui: ui,
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-rpc-addr", getRPCAddr(),
}
resultCh := make(chan int)
go func() {
resultCh <- c.Run(args)
}()
testutil.Yield()
// Verify it runs "forever"
select {
case <-resultCh:
t.Fatalf("ended too soon, err: %s", ui.ErrorWriter.String())
case <-time.After(50 * time.Millisecond):
}
// Send a shutdown request
shutdownCh <- struct{}{}
select {
case code := <-resultCh:
if code != 0 {
t.Fatalf("bad code: %d", code)
}
case <-time.After(50 * time.Millisecond):
t.Fatalf("timeout")
}
}
func TestCommandRun_join(t *testing.T) {
a1 := testAgent(nil)
if err := a1.Start(); err != nil {
t.Fatalf("err: %s", err)
}
defer a1.Shutdown()
doneCh := make(chan struct{})
shutdownCh := make(chan struct{})
defer func() {
close(shutdownCh)
<-doneCh
}()
c := &Command{
ShutdownCh: shutdownCh,
Ui: new(cli.MockUi),
}
args := []string{
"-bind", testutil.GetBindAddr().String(),
"-join", a1.conf.MemberlistConfig.BindAddr,
"-replay",
}
go func() {
code := c.Run(args)
if code != 0 {
log.Printf("bad: %d", code)
}
close(doneCh)
}()
testutil.Yield()
if len(a1.Serf().Members()) != 2 {
t.Fatalf("bad: %#v", a1.Serf().Members())
}
}
func Test_processFilteredNodes(t *testing.T) {
log.Level = logrus.ErrorLevel
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
s := NewStore("etcd", []string{}, nil, "dkron")
err := s.Client.DeleteTree("dkron")
if err != nil {
if err == store.ErrNotReachable {
t.Fatal("etcd server needed to run tests")
}
}
a1Addr := testutil.GetBindAddr().String()
a2Addr := testutil.GetBindAddr().String()
args := []string{
"-bind", a1Addr,
"-join", a2Addr,
"-node", "test1",
"-server",
"-tag", "role=test",
}
resultCh := make(chan int)
go func() {
resultCh <- a.Run(args)
}()
time.Sleep(2 * time.Second)
// Start another agent
shutdownCh2 := make(chan struct{})
defer close(shutdownCh2)
ui2 := new(cli.MockUi)
a2 := &AgentCommand{
Ui: ui2,
ShutdownCh: shutdownCh2,
}
args2 := []string{
"-bind", a2Addr,
"-join", a1Addr,
"-node", "test2",
"-server",
"-tag", "role=test",
}
resultCh2 := make(chan int)
go func() {
resultCh2 <- a2.Run(args2)
}()
job := &Job{
Name: "test_job_1",
Tags: map[string]string{
"role": "test:2",
},
}
time.Sleep(2 * time.Second)
nodes, err := a.processFilteredNodes(job)
if nodes[0] != "test1" || nodes[1] != "test2" {
t.Fatal("Not expected returned nodes")
}
// Send a shutdown request
shutdownCh <- struct{}{}
shutdownCh2 <- struct{}{}
}
func TestAgentCommandElectLeader(t *testing.T) {
log.Level = logrus.ErrorLevel
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a1 := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
s := NewStore("etcd", []string{}, nil, "dkron")
err := s.Client.DeleteTree("dkron")
if err != nil {
if err == store.ErrNotReachable {
t.Fatal("etcd server needed to run tests")
}
}
a1Addr := testutil.GetBindAddr().String()
a2Addr := testutil.GetBindAddr().String()
args := []string{
"-bind", a1Addr,
"-join", a2Addr,
"-node", "test1",
"-server",
}
resultCh := make(chan int)
go func() {
resultCh <- a1.Run(args)
}()
// Wait for the first agent to start and set itself as leader
time.Sleep(2 * time.Second)
test1Key := a1.config.Tags["key"]
t.Logf("test1 key %s", test1Key)
// Start another agent
shutdownCh2 := make(chan struct{})
defer close(shutdownCh2)
ui2 := new(cli.MockUi)
a2 := &AgentCommand{
Ui: ui2,
ShutdownCh: shutdownCh2,
}
defer func() { shutdownCh2 <- struct{}{} }()
args2 := []string{
"-bind", a2Addr,
"-join", a1Addr,
"-node", "test2",
"-server",
}
resultCh2 := make(chan int)
go func() {
resultCh2 <- a2.Run(args2)
}()
time.Sleep(2 * time.Second)
test2Key := a2.config.Tags["key"]
t.Logf("test2 key %s", test2Key)
// Send a shutdown request
shutdownCh <- struct{}{}
time.Sleep(2 * time.Second)
// Listen for leader key changes or timeout
stopCh := make(chan struct{})
receiver, err := s.Client.Watch("/dkron/leader", stopCh)
if err != nil {
t.Fatal(err)
}
// Wait for the new leader election
for {
select {
case res := <-receiver:
if res != nil {
if bytes.Equal(res.Value, []byte(test2Key)) {
t.Logf("Leader changed: %s", res.Value)
stopCh <- struct{}{}
return
}
if bytes.Equal(res.Value, []byte(test1Key)) {
t.Logf("Leader set to agent1: %s", res.Value)
}
}
case <-time.After(10 * time.Second):
t.Fatal("No leader swap occurred")
stopCh <- struct{}{}
return
}
}
}
func TestRPCExecutionDone(t *testing.T) {
store := NewStore("etcd", []string{"127.0.0.1:4001"}, nil, "dkron")
// Cleanup everything
err := store.Client.DeleteTree("dkron")
if err != nil {
t.Logf("error cleaning up: %s", err)
}
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
aAddr := testutil.GetBindAddr().String()
args := []string{
"-bind", aAddr,
"-node", "test1",
"-server",
"-keyspace", "dkron",
}
resultCh := make(chan int)
go func() {
resultCh <- a.Run(args)
}()
time.Sleep(2 * time.Second)
testJob := &Job{
Name: "test",
Schedule: "@every 2s",
Disabled: true,
}
if err := store.SetJob(testJob); err != nil {
t.Fatalf("error creating job: %s", err)
}
testExecution := &Execution{
JobName: "test",
Group: time.Now().UnixNano(),
StartedAt: time.Now(),
NodeName: "testNode",
FinishedAt: time.Now(),
Success: true,
Output: []byte("type"),
}
rc := &RPCClient{
ServerAddr: a.getRPCAddr(),
}
rc.callExecutionDone(testExecution)
execs, _ := store.GetExecutions("test")
if len(execs) == 0 {
t.Fatal("executions result is empty")
}
if string(execs[0].Output) != string(testExecution.Output) {
t.Fatalf("error on retrieved excution expected: %s got: %s", testExecution.Output, execs[0].Output)
}
// Test store execution on a deleted job
store.DeleteJob(testJob.Name)
testExecution.FinishedAt = time.Now()
rc.callExecutionDone(testExecution)
}
func TestCommandRun_mDNS(t *testing.T) {
// mDNS does not work in travis
if os.Getenv("TRAVIS") != "" {
t.SkipNow()
}
// Start an agent
doneCh := make(chan struct{})
shutdownCh := make(chan struct{})
defer func() {
close(shutdownCh)
<-doneCh
}()
c := &Command{
ShutdownCh: shutdownCh,
Ui: new(cli.MockUi),
}
args := []string{
"-node", "foo",
"-bind", testutil.GetBindAddr().String(),
"-discover", "test",
"-rpc-addr", getRPCAddr(),
}
go func() {
code := c.Run(args)
if code != 0 {
log.Printf("bad: %d", code)
}
close(doneCh)
}()
// Start a second agent
doneCh2 := make(chan struct{})
shutdownCh2 := make(chan struct{})
defer func() {
close(shutdownCh2)
<-doneCh2
}()
c2 := &Command{
ShutdownCh: shutdownCh2,
Ui: new(cli.MockUi),
}
addr2 := getRPCAddr()
args2 := []string{
"-node", "bar",
"-bind", testutil.GetBindAddr().String(),
"-discover", "test",
"-rpc-addr", addr2,
}
go func() {
code := c2.Run(args2)
if code != 0 {
log.Printf("bad: %d", code)
}
close(doneCh2)
}()
time.Sleep(150 * time.Millisecond)
client, err := client.NewRPCClient(addr2)
if err != nil {
t.Fatalf("err: %s", err)
}
defer client.Close()
members, err := client.Members()
if err != nil {
t.Fatalf("err: %s", err)
}
if len(members) != 2 {
t.Fatalf("bad: %#v", members)
}
}
func TestAgentCommand_runForElection(t *testing.T) {
a1Name := "test1"
a2Name := "test2"
a1Addr := testutil.GetBindAddr().String()
a2Addr := testutil.GetBindAddr().String()
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a1 := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
client, err := libkv.NewStore("etcd", []string{etcdAddr}, &store.Config{})
if err != nil {
panic(err)
}
err = client.DeleteTree("dkron")
if err != nil {
if err != store.ErrKeyNotFound {
panic(err)
}
}
args := []string{
"-bind", a1Addr,
"-join", a2Addr,
"-node", a1Name,
"-server",
"-log-level", logLevel,
}
resultCh := make(chan int)
go func() {
resultCh <- a1.Run(args)
}()
// Wait for the first agent to start and set itself as leader
time.Sleep(2 * time.Second)
// Start another agent
shutdownCh2 := make(chan struct{})
defer close(shutdownCh2)
ui2 := new(cli.MockUi)
a2 := &AgentCommand{
Ui: ui2,
ShutdownCh: shutdownCh2,
}
defer func() { shutdownCh2 <- struct{}{} }()
args2 := []string{
"-bind", a2Addr,
"-join", a1Addr + ":8946",
"-node", a2Name,
"-server",
"-log-level", logLevel,
}
resultCh2 := make(chan int)
go func() {
resultCh2 <- a2.Run(args2)
}()
kv, _ := client.Get("dkron/leader")
leader := string(kv.Value)
log.Printf("%s is the current leader", leader)
if leader != a1Name {
t.Errorf("Expected %s to be the leader, got %s", a1Name, leader)
}
// Send a shutdown request
shutdownCh <- struct{}{}
// Wait until test2 steps as leader
time.Sleep(30 * time.Second)
kv, _ = client.Get("dkron/leader")
leader = string(kv.Value)
log.Printf("%s is the current leader", leader)
if leader != a2Name {
t.Errorf("Expected %s to be the leader, got %s", a2Name, leader)
}
}
func Test_processFilteredNodes(t *testing.T) {
shutdownCh := make(chan struct{})
defer close(shutdownCh)
ui := new(cli.MockUi)
a := &AgentCommand{
Ui: ui,
ShutdownCh: shutdownCh,
}
client, err := libkv.NewStore("etcd", []string{etcdAddr}, &store.Config{})
err = client.DeleteTree("dkron")
if err != nil {
if err == store.ErrNotReachable {
t.Fatal("etcd server needed to run tests")
}
}
a1Addr := testutil.GetBindAddr().String()
a2Addr := testutil.GetBindAddr().String()
args := []string{
"-bind", a1Addr,
"-join", a2Addr,
"-node", "test1",
"-server",
"-tag", "role=test",
"-log-level", logLevel,
}
resultCh := make(chan int)
go func() {
resultCh <- a.Run(args)
}()
time.Sleep(2 * time.Second)
// Start another agent
shutdownCh2 := make(chan struct{})
defer close(shutdownCh2)
ui2 := new(cli.MockUi)
a2 := &AgentCommand{
Ui: ui2,
ShutdownCh: shutdownCh2,
}
args2 := []string{
"-bind", a2Addr,
"-join", a1Addr,
"-node", "test2",
"-server",
"-tag", "role=test",
"-log-level", logLevel,
}
resultCh2 := make(chan int)
go func() {
resultCh2 <- a2.Run(args2)
}()
job := &Job{
Name: "test_job_1",
Tags: map[string]string{
"foo": "bar:1",
"role": "test:2",
},
}
time.Sleep(2 * time.Second)
nodes, tags, err := a.processFilteredNodes(job)
assert.Contains(t, nodes, "test1")
assert.Contains(t, nodes, "test2")
assert.Equal(t, tags["role"], "test")
assert.Equal(t, job.Tags["role"], "test:2")
assert.Equal(t, job.Tags["foo"], "bar:1")
// Send a shutdown request
shutdownCh <- struct{}{}
shutdownCh2 <- struct{}{}
}