func TestFSM_SnapshotRestore_Allocs_NoSharedResources(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
alloc1 := mock.Alloc()
alloc2 := mock.Alloc()
alloc1.SharedResources = nil
alloc2.SharedResources = nil
state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID))
state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))
state.UpsertAllocs(1000, []*structs.Allocation{alloc1})
state.UpsertAllocs(1001, []*structs.Allocation{alloc2})
// Verify the contents
fsm2 := testSnapshotRestore(t, fsm)
state2 := fsm2.State()
out1, _ := state2.AllocByID(alloc1.ID)
out2, _ := state2.AllocByID(alloc2.ID)
alloc1.SharedResources = &structs.Resources{DiskMB: 150}
alloc2.SharedResources = &structs.Resources{DiskMB: 150}
if !reflect.DeepEqual(alloc1, out1) {
t.Fatalf("bad: \n%#v\n%#v", out1, alloc1)
}
if !reflect.DeepEqual(alloc2, out2) {
t.Fatalf("bad: \n%#v\n%#v", out2, alloc2)
}
}
func TestFSM_UpdateAllocFromClient(t *testing.T) {
fsm := testFSM(t)
state := fsm.State()
alloc := mock.Alloc()
state.UpsertAllocs(1, []*structs.Allocation{alloc})
clientAlloc := new(structs.Allocation)
*clientAlloc = *alloc
clientAlloc.ClientStatus = structs.AllocClientStatusFailed
req := structs.AllocUpdateRequest{
Alloc: []*structs.Allocation{clientAlloc},
}
buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := fsm.Apply(makeLog(buf))
if resp != nil {
t.Fatalf("resp: %v", resp)
}
// Verify we are registered
out, err := fsm.State().AllocByID(alloc.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
clientAlloc.CreateIndex = out.CreateIndex
clientAlloc.ModifyIndex = out.ModifyIndex
if !reflect.DeepEqual(clientAlloc, out) {
t.Fatalf("bad: %#v %#v", clientAlloc, out)
}
}
func TestFSM_SnapshotRestore_Allocs(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
alloc1 := mock.Alloc()
state.UpsertAllocs(1000, []*structs.Allocation{alloc1})
alloc2 := mock.Alloc()
state.UpsertAllocs(1001, []*structs.Allocation{alloc2})
// Verify the contents
fsm2 := testSnapshotRestore(t, fsm)
state2 := fsm2.State()
out1, _ := state2.AllocByID(alloc1.ID)
out2, _ := state2.AllocByID(alloc2.ID)
if !reflect.DeepEqual(alloc1, out1) {
t.Fatalf("bad: \n%#v\n%#v", out1, alloc1)
}
if !reflect.DeepEqual(alloc2, out2) {
t.Fatalf("bad: \n%#v\n%#v", out2, alloc2)
}
}
func TestInplaceUpdate_ChangedTaskGroup(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(900, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
TaskGroup: "web",
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
noErr(t, state.UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)))
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
// Create a new task group that prevents in-place updates.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
task := &structs.Task{Name: "FOO"}
tg.Tasks = nil
tg.Tasks = append(tg.Tasks, task)
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
// Do the inplace update.
unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 1 || len(inplace) != 0 {
t.Fatal("inplaceUpdate incorrectly did an inplace update")
}
if len(ctx.plan.NodeAllocation) != 0 {
t.Fatal("inplaceUpdate incorrectly did an inplace update")
}
}
func TestInplaceUpdate_Success(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(1000, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
// Create a new task group that updates the resources.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
resource := &structs.Resources{CPU: 737}
tg.Tasks[0].Resources = resource
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
stack.SetJob(job)
// Do the inplace update.
unplaced := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 0 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if len(ctx.plan.NodeAllocation) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
}
func TestFSM_SnapshotRestore_AddMissingSummary(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
// make an allocation
alloc := mock.Alloc()
state.UpsertJob(1010, alloc.Job)
state.UpsertAllocs(1011, []*structs.Allocation{alloc})
// Delete the summary
state.DeleteJobSummary(1040, alloc.Job.ID)
// Delete the index
if err := state.RemoveIndex("job_summary"); err != nil {
t.Fatalf("err: %v", err)
}
fsm2 := testSnapshotRestore(t, fsm)
state2 := fsm2.State()
latestIndex, _ := state.LatestIndex()
out, _ := state2.JobSummaryByID(alloc.Job.ID)
expected := structs.JobSummary{
JobID: alloc.Job.ID,
Summary: map[string]structs.TaskGroupSummary{
"web": structs.TaskGroupSummary{
Starting: 1,
},
},
CreateIndex: 1010,
ModifyIndex: latestIndex,
}
if !reflect.DeepEqual(&expected, out) {
t.Fatalf("expected: %#v, actual: %#v", &expected, out)
}
}
func TestInplaceUpdate_Success(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(900, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
TaskGroup: job.TaskGroups[0].Name,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
noErr(t, state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)))
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
// Create a new task group that updates the resources.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
resource := &structs.Resources{CPU: 737}
tg.Tasks[0].Resources = resource
newServices := []*structs.Service{
{
Name: "dummy-service",
PortLabel: "http",
},
{
Name: "dummy-service2",
PortLabel: "http",
},
}
// Delete service 2
tg.Tasks[0].Services = tg.Tasks[0].Services[:1]
// Add the new services
tg.Tasks[0].Services = append(tg.Tasks[0].Services, newServices...)
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
stack.SetJob(job)
// Do the inplace update.
unplaced, inplace := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 0 || len(inplace) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if len(ctx.plan.NodeAllocation) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if inplace[0].Alloc.ID != alloc.ID {
t.Fatalf("inplaceUpdate returned the wrong, inplace updated alloc: %#v", inplace)
}
// Get the alloc we inserted.
a := inplace[0].Alloc // TODO(sean@): Verify this is correct vs: ctx.plan.NodeAllocation[alloc.NodeID][0]
if a.Job == nil {
t.Fatalf("bad")
}
if len(a.Job.TaskGroups) != 1 {
t.Fatalf("bad")
}
if len(a.Job.TaskGroups[0].Tasks) != 1 {
t.Fatalf("bad")
}
if len(a.Job.TaskGroups[0].Tasks[0].Services) != 3 {
t.Fatalf("Expected number of services: %v, Actual: %v", 3, len(a.Job.TaskGroups[0].Tasks[0].Services))
}
serviceNames := make(map[string]struct{}, 3)
for _, consulService := range a.Job.TaskGroups[0].Tasks[0].Services {
serviceNames[consulService.Name] = struct{}{}
}
if len(serviceNames) != 3 {
t.Fatalf("bad")
}
for _, name := range []string{"dummy-service", "dummy-service2", "web-frontend"} {
if _, found := serviceNames[name]; !found {
t.Errorf("Expected consul service name missing: %v", name)
}
}
}
func TestEvalContext_ProposedAlloc(t *testing.T) {
state, ctx := testContext(t)
nodes := []*RankedNode{
&RankedNode{
Node: &structs.Node{
// Perfect fit
ID: structs.GenerateUUID(),
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
},
},
&RankedNode{
Node: &structs.Node{
// Perfect fit
ID: structs.GenerateUUID(),
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
},
},
}
// Add existing allocations
alloc1 := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: structs.GenerateUUID(),
NodeID: nodes[0].Node.ID,
JobID: structs.GenerateUUID(),
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
ClientStatus: structs.AllocClientStatusPending,
}
alloc2 := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: structs.GenerateUUID(),
NodeID: nodes[1].Node.ID,
JobID: structs.GenerateUUID(),
Resources: &structs.Resources{
CPU: 1024,
MemoryMB: 1024,
},
DesiredStatus: structs.AllocDesiredStatusRun,
ClientStatus: structs.AllocClientStatusPending,
}
noErr(t, state.UpsertAllocs(1000, []*structs.Allocation{alloc1, alloc2}))
// Add a planned eviction to alloc1
plan := ctx.Plan()
plan.NodeUpdate[nodes[0].Node.ID] = []*structs.Allocation{alloc1}
// Add a planned placement to node1
plan.NodeAllocation[nodes[1].Node.ID] = []*structs.Allocation{
&structs.Allocation{
Resources: &structs.Resources{
CPU: 1024,
MemoryMB: 1024,
},
},
}
proposed, err := ctx.ProposedAllocs(nodes[0].Node.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(proposed) != 0 {
t.Fatalf("bad: %#v", proposed)
}
proposed, err = ctx.ProposedAllocs(nodes[1].Node.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
if len(proposed) != 2 {
t.Fatalf("bad: %#v", proposed)
}
}
func TestInplaceUpdate_Success(t *testing.T) {
state, ctx := testContext(t)
eval := mock.Eval()
job := mock.Job()
node := mock.Node()
noErr(t, state.UpsertNode(1000, node))
// Register an alloc
alloc := &structs.Allocation{
ID: structs.GenerateUUID(),
EvalID: eval.ID,
NodeID: node.ID,
JobID: job.ID,
Job: job,
TaskGroup: job.TaskGroups[0].Name,
Resources: &structs.Resources{
CPU: 2048,
MemoryMB: 2048,
},
DesiredStatus: structs.AllocDesiredStatusRun,
}
alloc.TaskResources = map[string]*structs.Resources{"web": alloc.Resources}
alloc.PopulateServiceIDs()
noErr(t, state.UpsertAllocs(1001, []*structs.Allocation{alloc}))
webFeSrvID := alloc.Services["web-frontend"]
adminSrvID := alloc.Services["web-admin"]
if webFeSrvID == "" || adminSrvID == "" {
t.Fatal("Service ID needs to be generated for service")
}
// Create a new task group that updates the resources.
tg := &structs.TaskGroup{}
*tg = *job.TaskGroups[0]
resource := &structs.Resources{CPU: 737}
tg.Tasks[0].Resources = resource
newServices := []*structs.Service{
{
Name: "dummy-service",
PortLabel: "http",
},
{
Name: "dummy-service2",
PortLabel: "http",
},
}
// Delete service 2
tg.Tasks[0].Services = tg.Tasks[0].Services[:1]
// Add the new services
tg.Tasks[0].Services = append(tg.Tasks[0].Services, newServices...)
updates := []allocTuple{{Alloc: alloc, TaskGroup: tg}}
stack := NewGenericStack(false, ctx)
stack.SetJob(job)
// Do the inplace update.
unplaced := inplaceUpdate(ctx, eval, job, stack, updates)
if len(unplaced) != 0 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
if len(ctx.plan.NodeAllocation) != 1 {
t.Fatal("inplaceUpdate did not do an inplace update")
}
// Get the alloc we inserted.
a := ctx.plan.NodeAllocation[alloc.NodeID][0]
if len(a.Services) != 3 {
t.Fatalf("Expected number of services: %v, Actual: %v", 3, len(a.Services))
}
// Test that the service id for the old service is still the same
if a.Services["web-frontend"] != webFeSrvID {
t.Fatalf("Expected service ID: %v, Actual: %v", webFeSrvID, a.Services["web-frontend"])
}
// Test that the map doesn't contain the service ID of the admin Service
// anymore
if _, ok := a.Services["web-admin"]; ok {
t.Fatal("Service shouldn't be present")
}
}
func TestFSM_UpdateAllocFromClient(t *testing.T) {
fsm := testFSM(t)
fsm.blockedEvals.SetEnabled(true)
state := fsm.State()
node := mock.Node()
state.UpsertNode(1, node)
// Mark an eval as blocked.
eval := mock.Eval()
eval.ClassEligibility = map[string]bool{node.ComputedClass: true}
fsm.blockedEvals.Block(eval)
bStats := fsm.blockedEvals.Stats()
if bStats.TotalBlocked != 1 {
t.Fatalf("bad: %#v", bStats)
}
// Create a completed eval
alloc := mock.Alloc()
alloc.NodeID = node.ID
state.UpsertAllocs(1, []*structs.Allocation{alloc})
clientAlloc := new(structs.Allocation)
*clientAlloc = *alloc
clientAlloc.ClientStatus = structs.AllocClientStatusDead
req := structs.AllocUpdateRequest{
Alloc: []*structs.Allocation{clientAlloc},
}
buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := fsm.Apply(makeLog(buf))
if resp != nil {
t.Fatalf("resp: %v", resp)
}
// Verify we are registered
out, err := fsm.State().AllocByID(alloc.ID)
if err != nil {
t.Fatalf("err: %v", err)
}
clientAlloc.CreateIndex = out.CreateIndex
clientAlloc.ModifyIndex = out.ModifyIndex
if !reflect.DeepEqual(clientAlloc, out) {
t.Fatalf("bad: %#v %#v", clientAlloc, out)
}
// Verify the eval was unblocked.
testutil.WaitForResult(func() (bool, error) {
bStats = fsm.blockedEvals.Stats()
if bStats.TotalBlocked != 0 {
return false, fmt.Errorf("bad: %#v %#v", bStats, out)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestFSM_ReconcileSummaries(t *testing.T) {
// Add some state
fsm := testFSM(t)
state := fsm.State()
// Add a node
node := mock.Node()
state.UpsertNode(800, node)
// Make a job so that none of the tasks can be placed
job1 := mock.Job()
job1.TaskGroups[0].Tasks[0].Resources.CPU = 5000
state.UpsertJob(1000, job1)
// make a job which can make partial progress
alloc := mock.Alloc()
alloc.NodeID = node.ID
state.UpsertJob(1010, alloc.Job)
state.UpsertAllocs(1011, []*structs.Allocation{alloc})
// Delete the summaries
state.DeleteJobSummary(1030, job1.ID)
state.DeleteJobSummary(1040, alloc.Job.ID)
req := structs.GenericRequest{}
buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := fsm.Apply(makeLog(buf))
if resp != nil {
t.Fatalf("resp: %v", resp)
}
out1, _ := state.JobSummaryByID(job1.ID)
expected := structs.JobSummary{
JobID: job1.ID,
Summary: map[string]structs.TaskGroupSummary{
"web": structs.TaskGroupSummary{
Queued: 10,
},
},
CreateIndex: 1000,
ModifyIndex: out1.ModifyIndex,
}
if !reflect.DeepEqual(&expected, out1) {
t.Fatalf("expected: %#v, actual: %#v", &expected, out1)
}
// This exercises the code path which adds the allocations made by the
// planner and the number of unplaced allocations in the reconcile summaries
// codepath
out2, _ := state.JobSummaryByID(alloc.Job.ID)
expected = structs.JobSummary{
JobID: alloc.Job.ID,
Summary: map[string]structs.TaskGroupSummary{
"web": structs.TaskGroupSummary{
Queued: 10,
Starting: 1,
},
},
CreateIndex: 1010,
ModifyIndex: out2.ModifyIndex,
}
if !reflect.DeepEqual(&expected, out2) {
t.Fatalf("expected: %#v, actual: %#v", &expected, out2)
}
}