func TestReadyNodesInDCs(t *testing.T) {
state, err := state.NewStateStore(os.Stderr)
if err != nil {
t.Fatalf("err: %v", err)
}
node1 := mock.Node()
node2 := mock.Node()
node2.Datacenter = "dc2"
node3 := mock.Node()
node3.Datacenter = "dc2"
node3.Status = structs.NodeStatusDown
node4 := mock.Node()
node4.Drain = true
noErr(t, state.UpsertNode(1000, node1))
noErr(t, state.UpsertNode(1001, node2))
noErr(t, state.UpsertNode(1002, node3))
noErr(t, state.UpsertNode(1003, node4))
nodes, err := readyNodesInDCs(state, []string{"dc1", "dc2"})
if err != nil {
t.Fatalf("err: %v", err)
}
if len(nodes) != 2 {
t.Fatalf("bad: %v", nodes)
}
if nodes[0].ID == node3.ID || nodes[1].ID == node3.ID {
t.Fatalf("Bad: %#v", nodes)
}
}
func TestTaintedNodes(t *testing.T) {
state, err := state.NewStateStore(os.Stderr)
if err != nil {
t.Fatalf("err: %v", err)
}
node1 := mock.Node()
node2 := mock.Node()
node2.Datacenter = "dc2"
node3 := mock.Node()
node3.Datacenter = "dc2"
node3.Status = structs.NodeStatusDown
node4 := mock.Node()
node4.Drain = true
noErr(t, state.UpsertNode(1000, node1))
noErr(t, state.UpsertNode(1001, node2))
noErr(t, state.UpsertNode(1002, node3))
noErr(t, state.UpsertNode(1003, node4))
allocs := []*structs.Allocation{
&structs.Allocation{NodeID: node1.ID},
&structs.Allocation{NodeID: node2.ID},
&structs.Allocation{NodeID: node3.ID},
&structs.Allocation{NodeID: node4.ID},
&structs.Allocation{NodeID: "12345678-abcd-efab-cdef-123456789abc"},
}
tainted, err := taintedNodes(state, allocs)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(tainted) != 3 {
t.Fatalf("bad: %v", tainted)
}
if _, ok := tainted[node1.ID]; ok {
t.Fatalf("Bad: %v", tainted)
}
if _, ok := tainted[node2.ID]; ok {
t.Fatalf("Bad: %v", tainted)
}
if node, ok := tainted[node3.ID]; !ok || node == nil {
t.Fatalf("Bad: %v", tainted)
}
if node, ok := tainted[node4.ID]; !ok || node == nil {
t.Fatalf("Bad: %v", tainted)
}
if node, ok := tainted["12345678-abcd-efab-cdef-123456789abc"]; !ok || node != nil {
t.Fatalf("Bad: %v", tainted)
}
}
func TestTaintedNodes(t *testing.T) {
state, err := state.NewStateStore(os.Stderr)
if err != nil {
t.Fatalf("err: %v", err)
}
node1 := mock.Node()
node2 := mock.Node()
node2.Datacenter = "dc2"
node3 := mock.Node()
node3.Datacenter = "dc2"
node3.Status = structs.NodeStatusDown
node4 := mock.Node()
node4.Drain = true
noErr(t, state.UpsertNode(1000, node1))
noErr(t, state.UpsertNode(1001, node2))
noErr(t, state.UpsertNode(1002, node3))
noErr(t, state.UpsertNode(1003, node4))
allocs := []*structs.Allocation{
&structs.Allocation{NodeID: node1.ID},
&structs.Allocation{NodeID: node2.ID},
&structs.Allocation{NodeID: node3.ID},
&structs.Allocation{NodeID: node4.ID},
&structs.Allocation{NodeID: "blah"},
}
tainted, err := taintedNodes(state, allocs)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(tainted) != 5 {
t.Fatalf("bad: %v", tainted)
}
if tainted[node1.ID] || tainted[node2.ID] {
t.Fatalf("Bad: %v", tainted)
}
if !tainted[node3.ID] || !tainted[node4.ID] || !tainted["blah"] {
t.Fatalf("Bad: %v", tainted)
}
}
func TestFSM_SnapshotRestore_Nodes(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
node1 := mock.Node()
state.UpsertNode(1000, node1)
node2 := mock.Node()
state.UpsertNode(1001, node2)
// Verify the contents
fsm2 := testSnapshotRestore(t, fsm)
state2 := fsm2.State()
out1, _ := state2.NodeByID(node1.ID)
out2, _ := state2.NodeByID(node2.ID)
if !reflect.DeepEqual(node1, out1) {
t.Fatalf("bad: \n%#v\n%#v", out1, node1)
}
if !reflect.DeepEqual(node2, out2) {
t.Fatalf("bad: \n%#v\n%#v", out2, node2)
}
}
func TestInplaceUpdate_ChangedTaskGroup(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(900, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
TaskGroup: "web",
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
noErr(t, state.UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)))
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
// Create a new task group that prevents in-place updates.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
task := &structs.Task{Name: "FOO"}
tg.Tasks = nil
tg.Tasks = append(tg.Tasks, task)
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
// Do the inplace update.
unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 1 || len(inplace) != 0 {
t.Fatal("inplaceUpdate incorrectly did an inplace update")
}
if len(ctx.plan.NodeAllocation) != 0 {
t.Fatal("inplaceUpdate incorrectly did an inplace update")
}
}
func TestInplaceUpdate_Success(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(1000, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
// Create a new task group that updates the resources.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
resource := &structs.Resources{CPU: 737}
tg.Tasks[0].Resources = resource
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
stack.SetJob(job)
// Do the inplace update.
unplaced := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 0 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if len(ctx.plan.NodeAllocation) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
}
func TestFSM_SnapshotRestore_Indexes(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
node1 := mock.Node()
state.UpsertNode(1000, node1)
// Verify the contents
fsm2 := testSnapshotRestore(t, fsm)
state2 := fsm2.State()
index, err := state2.Index("nodes")
if err != nil {
t.Fatalf("err: %v", err)
}
if index != 1000 {
t.Fatalf("bad: %d", index)
}
}
func TestInplaceUpdate_Success(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(900, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
TaskGroup: job.TaskGroups[0].Name,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
noErr(t, state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)))
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
// Create a new task group that updates the resources.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
resource := &structs.Resources{CPU: 737}
tg.Tasks[0].Resources = resource
newServices := []*structs.Service{
{
Name: "dummy-service",
PortLabel: "http",
},
{
Name: "dummy-service2",
PortLabel: "http",
},
}
// Delete service 2
tg.Tasks[0].Services = tg.Tasks[0].Services[:1]
// Add the new services
tg.Tasks[0].Services = append(tg.Tasks[0].Services, newServices...)
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
stack.SetJob(job)
// Do the inplace update.
unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 0 || len(inplace) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if len(ctx.plan.NodeAllocation) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if inplace[0].Alloc.ID != alloc.ID {
t.Fatalf("inplaceUpdate returned the wrong, inplace updated alloc: %#v", inplace)
}
// Get the alloc we inserted.
a := inplace[0].Alloc // TODO(sean@): Verify this is correct vs: ctx.plan.NodeAllocation[alloc.NodeID][0]
if a.Job == nil {
t.Fatalf("bad")
}
if len(a.Job.TaskGroups) != 1 {
t.Fatalf("bad")
}
if len(a.Job.TaskGroups[0].Tasks) != 1 {
t.Fatalf("bad")
}
if len(a.Job.TaskGroups[0].Tasks[0].Services) != 3 {
t.Fatalf("Expected number of services: %v, Actual: %v", 3, len(a.Job.TaskGroups[0].Tasks[0].Services))
}
serviceNames := make(map[string]struct{}, 3)
for _, consulService := range a.Job.TaskGroups[0].Tasks[0].Services {
serviceNames[consulService.Name] = struct{}{}
}
if len(serviceNames) != 3 {
t.Fatalf("bad")
}
for _, name := range []string{"dummy-service", "dummy-service2", "web-frontend"} {
if _, found := serviceNames[name]; !found {
t.Errorf("Expected consul service name missing: %v", name)
}
}
}
func TestInplaceUpdate_Success(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(1000, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
TaskGroup: job.TaskGroups[0].Name,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
alloc.PopulateServiceIDs()
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
webFeSrvID := alloc.Services["web-frontend"]
adminSrvID := alloc.Services["web-admin"]
if webFeSrvID == "" || adminSrvID == "" {
t.Fatal("Service ID needs to be generated for service")
}
// Create a new task group that updates the resources.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
resource := &structs.Resources{CPU: 737}
tg.Tasks[0].Resources = resource
newServices := []*structs.Service{
{
Name: "dummy-service",
PortLabel: "http",
},
{
Name: "dummy-service2",
PortLabel: "http",
},
}
// Delete service 2
tg.Tasks[0].Services = tg.Tasks[0].Services[:1]
// Add the new services
tg.Tasks[0].Services = append(tg.Tasks[0].Services, newServices...)
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
stack.SetJob(job)
// Do the inplace update.
unplaced := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 0 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if len(ctx.plan.NodeAllocation) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
// Get the alloc we inserted.
a := ctx.plan.NodeAllocation[alloc.NodeID][0]
if len(a.Services) != 3 {
t.Fatalf("Expected number of services: %v, Actual: %v", 3, len(a.Services))
}
// Test that the service id for the old service is still the same
if a.Services["web-frontend"] != webFeSrvID {
t.Fatalf("Expected service ID: %v, Actual: %v", webFeSrvID, a.Services["web-frontend"])
}
// Test that the map doesn't contain the service ID of the admin Service
// anymore
if _, ok := a.Services["web-admin"]; ok {
t.Fatal("Service shouldn't be present")
}
}
func TestFSM_UpdateAllocFromClient(t *testing.T) {
fsm := testFSM(t)
fsm.blockedEvals.SetEnabled(true)
state := fsm.State()
node := mock.Node()
state.UpsertNode(1, node)
// Mark an eval as blocked.
eval := mock.Eval()
eval.ClassEligibility = map[string]bool{node.ComputedClass: true}
fsm.blockedEvals.Block(eval)
bStats := fsm.blockedEvals.Stats()
if bStats.TotalBlocked != 1 {
t.Fatalf("bad: %#v", bStats)
}
// Create a completed eval
alloc := mock.Alloc()
alloc.NodeID = node.ID
state.UpsertAllocs(1, []*structs.Allocation{alloc})
clientAlloc := new(structs.Allocation)
*clientAlloc = *alloc
clientAlloc.ClientStatus = structs.AllocClientStatusDead
req := structs.AllocUpdateRequest{
Alloc: []*structs.Allocation{clientAlloc},
}
buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := fsm.Apply(makeLog(buf))
if resp != nil {
t.Fatalf("resp: %v", resp)
}
// Verify we are registered
out, err := fsm.State().AllocByID(alloc.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
clientAlloc.CreateIndex = out.CreateIndex
clientAlloc.ModifyIndex = out.ModifyIndex
if !reflect.DeepEqual(clientAlloc, out) {
t.Fatalf("bad: %#v %#v", clientAlloc, out)
}
// Verify the eval was unblocked.
testutil.WaitForResult(func() (bool, error) {
bStats = fsm.blockedEvals.Stats()
if bStats.TotalBlocked != 0 {
return false, fmt.Errorf("bad: %#v %#v", bStats, out)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestFSM_ReconcileSummaries(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
// Add a node
node := mock.Node()
state.UpsertNode(800, node)
// Make a job so that none of the tasks can be placed
job1 := mock.Job()
job1.TaskGroups[0].Tasks[0].Resources.CPU = 5000
state.UpsertJob(1000, job1)
// make a job which can make partial progress
alloc := mock.Alloc()
alloc.NodeID = node.ID
state.UpsertJob(1010, alloc.Job)
state.UpsertAllocs(1011, []*structs.Allocation{alloc})
// Delete the summaries
state.DeleteJobSummary(1030, job1.ID)
state.DeleteJobSummary(1040, alloc.Job.ID)
req := structs.GenericRequest{}
buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := fsm.Apply(makeLog(buf))
if resp != nil {
t.Fatalf("resp: %v", resp)
}
out1, _ := state.JobSummaryByID(job1.ID)
expected := structs.JobSummary{
JobID: job1.ID,
Summary: map[string]structs.TaskGroupSummary{
"web": structs.TaskGroupSummary{
Queued: 10,
},
},
CreateIndex: 1000,
ModifyIndex: out1.ModifyIndex,
}
if !reflect.DeepEqual(&expected, out1) {
t.Fatalf("expected: %#v, actual: %#v", &expected, out1)
}
// This exercises the code path which adds the allocations made by the
// planner and the number of unplaced allocations in the reconcile summaries
// codepath
out2, _ := state.JobSummaryByID(alloc.Job.ID)
expected = structs.JobSummary{
JobID: alloc.Job.ID,
Summary: map[string]structs.TaskGroupSummary{
"web": structs.TaskGroupSummary{
Queued: 10,
Starting: 1,
},
},
CreateIndex: 1010,
ModifyIndex: out2.ModifyIndex,
}
if !reflect.DeepEqual(&expected, out2) {
t.Fatalf("expected: %#v, actual: %#v", &expected, out2)
}
}