func TestFSM_UpdateNodeStatus(t *testing.T) { fsm := testFSM(t) fsm.blockedEvals.SetEnabled(true) node := mock.Node() req := structs.NodeRegisterRequest{ Node: node, } buf, err := structs.Encode(structs.NodeRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Mark an eval as blocked. eval := mock.Eval() eval.ClassEligibility = map[string]bool{node.ComputedClass: true} fsm.blockedEvals.Block(eval) req2 := structs.NodeUpdateStatusRequest{ NodeID: node.ID, Status: structs.NodeStatusReady, } buf, err = structs.Encode(structs.NodeUpdateStatusRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify the status is ready. node, err = fsm.State().NodeByID(req.Node.ID) if err != nil { t.Fatalf("err: %v", err) } if node.Status != structs.NodeStatusReady { t.Fatalf("bad node: %#v", node) } // Verify the eval was unblocked. testutil.WaitForResult(func() (bool, error) { bStats := fsm.blockedEvals.Stats() if bStats.TotalBlocked != 0 { return false, fmt.Errorf("bad: %#v", bStats) } return true, nil }, func(err error) { t.Fatalf("err: %s", err) }) }
func TestFSM_UpsertAllocs(t *testing.T) { fsm := testFSM(t) alloc := mock.Alloc() fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) req := structs.AllocUpdateRequest{ Alloc: []*structs.Allocation{alloc}, } buf, err := structs.Encode(structs.AllocUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out, err := fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } alloc.CreateIndex = out.CreateIndex alloc.ModifyIndex = out.ModifyIndex alloc.AllocModifyIndex = out.AllocModifyIndex if !reflect.DeepEqual(alloc, out) { t.Fatalf("bad: %#v %#v", alloc, out) } evictAlloc := new(structs.Allocation) *evictAlloc = *alloc evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict req2 := structs.AllocUpdateRequest{ Alloc: []*structs.Allocation{evictAlloc}, } buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are evicted out, err = fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } if out.DesiredStatus != structs.AllocDesiredStatusEvict { t.Fatalf("alloc found!") } }
func TestFSM_DeregisterJob(t *testing.T) { fsm := testFSM(t) job := mock.PeriodicJob() req := structs.JobRegisterRequest{ Job: job, } buf, err := structs.Encode(structs.JobRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } req2 := structs.JobDeregisterRequest{ JobID: job.ID, } buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are NOT registered jobOut, err := fsm.State().JobByID(req.Job.ID) if err != nil { t.Fatalf("err: %v", err) } if jobOut != nil { t.Fatalf("job found!") } // Verify it was removed from the periodic runner. if _, ok := fsm.periodicDispatcher.tracked[job.ID]; ok { t.Fatal("job not removed from periodic runner") } // Verify it was removed from the periodic launch table. launchOut, err := fsm.State().PeriodicLaunchByID(req.Job.ID) if err != nil { t.Fatalf("err: %v", err) } if launchOut != nil { t.Fatalf("launch found!") } }
func TestFSM_RegisterJob(t *testing.T) { fsm := testFSM(t) req := structs.JobRegisterRequest{ Job: mock.Job(), } buf, err := structs.Encode(structs.JobRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered job, err := fsm.State().JobByID(req.Job.ID) if err != nil { t.Fatalf("err: %v", err) } if job == nil { t.Fatalf("not found!") } if job.CreateIndex != 1 { t.Fatalf("bad index: %d", job.CreateIndex) } }
func TestFSM_UpsertNode(t *testing.T) { fsm := testFSM(t) req := structs.NodeRegisterRequest{ Node: mock.Node(), } buf, err := structs.Encode(structs.NodeRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered node, err := fsm.State().NodeByID(req.Node.ID) if err != nil { t.Fatalf("err: %v", err) } if node == nil { t.Fatalf("not found!") } if node.CreateIndex != 1 { t.Fatalf("bad index: %d", node.CreateIndex) } tt := fsm.TimeTable() index := tt.NearestIndex(time.Now().UTC()) if index != 1 { t.Fatalf("bad: %d", index) } }
func TestFSM_UpdateAllocFromClient(t *testing.T) { fsm := testFSM(t) state := fsm.State() alloc := mock.Alloc() state.UpsertAllocs(1, []*structs.Allocation{alloc}) clientAlloc := new(structs.Allocation) *clientAlloc = *alloc clientAlloc.ClientStatus = structs.AllocClientStatusFailed req := structs.AllocUpdateRequest{ Alloc: []*structs.Allocation{clientAlloc}, } buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out, err := fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } clientAlloc.CreateIndex = out.CreateIndex clientAlloc.ModifyIndex = out.ModifyIndex if !reflect.DeepEqual(clientAlloc, out) { t.Fatalf("bad: %#v %#v", clientAlloc, out) } }
func TestFSM_UpdateEval(t *testing.T) { fsm := testFSM(t) fsm.evalBroker.SetEnabled(true) req := structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{mock.Eval()}, } buf, err := structs.Encode(structs.EvalUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered eval, err := fsm.State().EvalByID(req.Evals[0].ID) if err != nil { t.Fatalf("err: %v", err) } if eval == nil { t.Fatalf("not found!") } if eval.CreateIndex != 1 { t.Fatalf("bad index: %d", eval.CreateIndex) } // Verify enqueued stats := fsm.evalBroker.Stats() if stats.TotalReady != 1 { t.Fatalf("bad: %#v %#v", stats, eval) } }
func TestFSM_UpsertNode(t *testing.T) { fsm := testFSM(t) fsm.blockedEvals.SetEnabled(true) node := mock.Node() // Mark an eval as blocked. eval := mock.Eval() eval.ClassEligibility = map[string]bool{node.ComputedClass: true} fsm.blockedEvals.Block(eval) req := structs.NodeRegisterRequest{ Node: node, } buf, err := structs.Encode(structs.NodeRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered n, err := fsm.State().NodeByID(req.Node.ID) if err != nil { t.Fatalf("err: %v", err) } if n == nil { t.Fatalf("not found!") } if n.CreateIndex != 1 { t.Fatalf("bad index: %d", node.CreateIndex) } tt := fsm.TimeTable() index := tt.NearestIndex(time.Now().UTC()) if index != 1 { t.Fatalf("bad: %d", index) } // Verify the eval was unblocked. testutil.WaitForResult(func() (bool, error) { bStats := fsm.blockedEvals.Stats() if bStats.TotalBlocked != 0 { return false, fmt.Errorf("bad: %#v", bStats) } return true, nil }, func(err error) { t.Fatalf("err: %s", err) }) }
func TestFSM_UpdateNodeDrain(t *testing.T) { fsm := testFSM(t) node := mock.Node() req := structs.NodeRegisterRequest{ Node: node, } buf, err := structs.Encode(structs.NodeRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } req2 := structs.NodeUpdateDrainRequest{ NodeID: node.ID, Drain: true, } buf, err = structs.Encode(structs.NodeUpdateDrainRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are NOT registered node, err = fsm.State().NodeByID(req.Node.ID) if err != nil { t.Fatalf("err: %v", err) } if !node.Drain { t.Fatalf("bad node: %#v", node) } }
func TestFSM_DeregisterJob(t *testing.T) { fsm := testFSM(t) job := mock.Job() req := structs.JobRegisterRequest{ Job: job, } buf, err := structs.Encode(structs.JobRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } req2 := structs.JobDeregisterRequest{ JobID: job.ID, } buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are NOT registered job, err = fsm.State().JobByID(req.Job.ID) if err != nil { t.Fatalf("err: %v", err) } if job != nil { t.Fatalf("job found!") } }
func TestFSM_DeleteEval(t *testing.T) { fsm := testFSM(t) eval := mock.Eval() req := structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, } buf, err := structs.Encode(structs.EvalUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } req2 := structs.EvalDeleteRequest{ Evals: []string{eval.ID}, } buf, err = structs.Encode(structs.EvalDeleteRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are NOT registered eval, err = fsm.State().EvalByID(req.Evals[0].ID) if err != nil { t.Fatalf("err: %v", err) } if eval != nil { t.Fatalf("eval found!") } }
func TestFSM_DeregisterNode(t *testing.T) { fsm := testFSM(t) node := mock.Node() req := structs.NodeRegisterRequest{ Node: node, } buf, err := structs.Encode(structs.NodeRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } req2 := structs.NodeDeregisterRequest{ NodeID: node.ID, } buf, err = structs.Encode(structs.NodeDeregisterRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are NOT registered node, err = fsm.State().NodeByID(req.Node.ID) if err != nil { t.Fatalf("err: %v", err) } if node != nil { t.Fatalf("node found!") } }
// raftApplyFuture is used to encode a message, run it through raft, and return the Raft future. func (s *Server) raftApplyFuture(t structs.MessageType, msg interface{}) (raft.ApplyFuture, error) { buf, err := structs.Encode(t, msg) if err != nil { return nil, fmt.Errorf("Failed to encode request: %v", err) } // Warn if the command is very large if n := len(buf); n > raftWarnSize { s.logger.Printf("[WARN] nomad: Attempting to apply large raft entry (type %d) (%d bytes)", t, n) } future := s.raft.Apply(buf, enqueueLimit) return future, nil }
func TestFSM_UpsertVaultAccessor(t *testing.T) { fsm := testFSM(t) fsm.blockedEvals.SetEnabled(true) va := mock.VaultAccessor() va2 := mock.VaultAccessor() req := structs.VaultAccessorsRequest{ Accessors: []*structs.VaultAccessor{va, va2}, } buf, err := structs.Encode(structs.VaultAccessorRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out1, err := fsm.State().VaultAccessor(va.Accessor) if err != nil { t.Fatalf("err: %v", err) } if out1 == nil { t.Fatalf("not found!") } if out1.CreateIndex != 1 { t.Fatalf("bad index: %d", out1.CreateIndex) } out2, err := fsm.State().VaultAccessor(va2.Accessor) if err != nil { t.Fatalf("err: %v", err) } if out2 == nil { t.Fatalf("not found!") } if out1.CreateIndex != 1 { t.Fatalf("bad index: %d", out2.CreateIndex) } tt := fsm.TimeTable() index := tt.NearestIndex(time.Now().UTC()) if index != 1 { t.Fatalf("bad: %d", index) } }
func TestFSM_UpdateEval_Blocked(t *testing.T) { fsm := testFSM(t) fsm.evalBroker.SetEnabled(true) fsm.blockedEvals.SetEnabled(true) // Create a blocked eval. eval := mock.Eval() eval.Status = structs.EvalStatusBlocked req := structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, } buf, err := structs.Encode(structs.EvalUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out, err := fsm.State().EvalByID(eval.ID) if err != nil { t.Fatalf("err: %v", err) } if out == nil { t.Fatalf("not found!") } if out.CreateIndex != 1 { t.Fatalf("bad index: %d", out.CreateIndex) } // Verify the eval wasn't enqueued stats := fsm.evalBroker.Stats() if stats.TotalReady != 0 { t.Fatalf("bad: %#v %#v", stats, out) } // Verify the eval was added to the blocked tracker. bStats := fsm.blockedEvals.Stats() if bStats.TotalBlocked != 1 { t.Fatalf("bad: %#v %#v", bStats, out) } }
func TestFSM_RegisterJob(t *testing.T) { fsm := testFSM(t) job := mock.PeriodicJob() req := structs.JobRegisterRequest{ Job: job, } buf, err := structs.Encode(structs.JobRegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered jobOut, err := fsm.State().JobByID(req.Job.ID) if err != nil { t.Fatalf("err: %v", err) } if jobOut == nil { t.Fatalf("not found!") } if jobOut.CreateIndex != 1 { t.Fatalf("bad index: %d", jobOut.CreateIndex) } // Verify it was added to the periodic runner. if _, ok := fsm.periodicDispatcher.tracked[job.ID]; !ok { t.Fatal("job not added to periodic runner") } // Verify the launch time was tracked. launchOut, err := fsm.State().PeriodicLaunchByID(req.Job.ID) if err != nil { t.Fatalf("err: %v", err) } if launchOut == nil { t.Fatalf("not found!") } if launchOut.Launch.IsZero() { t.Fatalf("bad launch time: %v", launchOut.Launch) } }
func TestFSM_DeregisterVaultAccessor(t *testing.T) { fsm := testFSM(t) fsm.blockedEvals.SetEnabled(true) va := mock.VaultAccessor() va2 := mock.VaultAccessor() accessors := []*structs.VaultAccessor{va, va2} // Insert the accessors if err := fsm.State().UpsertVaultAccessor(1000, accessors); err != nil { t.Fatalf("bad: %v", err) } req := structs.VaultAccessorsRequest{ Accessors: accessors, } buf, err := structs.Encode(structs.VaultAccessorDegisterRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } out1, err := fsm.State().VaultAccessor(va.Accessor) if err != nil { t.Fatalf("err: %v", err) } if out1 != nil { t.Fatalf("not deleted!") } tt := fsm.TimeTable() index := tt.NearestIndex(time.Now().UTC()) if index != 1 { t.Fatalf("bad: %d", index) } }
func TestFSM_UpsertAllocs_StrippedResources(t *testing.T) { fsm := testFSM(t) alloc := mock.Alloc() fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) job := alloc.Job resources := alloc.Resources alloc.Resources = nil req := structs.AllocUpdateRequest{ Job: job, Alloc: []*structs.Allocation{alloc}, } buf, err := structs.Encode(structs.AllocUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out, err := fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } alloc.CreateIndex = out.CreateIndex alloc.ModifyIndex = out.ModifyIndex alloc.AllocModifyIndex = out.AllocModifyIndex // Resources should be recomputed resources.DiskMB = alloc.Job.TaskGroups[0].LocalDisk.DiskMB alloc.Resources = resources if !reflect.DeepEqual(alloc, out) { t.Fatalf("bad: %#v %#v", alloc, out) } }
func TestFSM_ReconcileSummaries(t *testing.T) { // Add some state fsm := testFSM(t) state := fsm.State() // Add a node node := mock.Node() state.UpsertNode(800, node) // Make a job so that none of the tasks can be placed job1 := mock.Job() job1.TaskGroups[0].Tasks[0].Resources.CPU = 5000 state.UpsertJob(1000, job1) // make a job which can make partial progress alloc := mock.Alloc() alloc.NodeID = node.ID state.UpsertJob(1010, alloc.Job) state.UpsertAllocs(1011, []*structs.Allocation{alloc}) // Delete the summaries state.DeleteJobSummary(1030, job1.ID) state.DeleteJobSummary(1040, alloc.Job.ID) req := structs.GenericRequest{} buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } out1, _ := state.JobSummaryByID(job1.ID) expected := structs.JobSummary{ JobID: job1.ID, Summary: map[string]structs.TaskGroupSummary{ "web": structs.TaskGroupSummary{ Queued: 10, }, }, CreateIndex: 1000, ModifyIndex: out1.ModifyIndex, } if !reflect.DeepEqual(&expected, out1) { t.Fatalf("expected: %#v, actual: %#v", &expected, out1) } // This exercises the code path which adds the allocations made by the // planner and the number of unplaced allocations in the reconcile summaries // codepath out2, _ := state.JobSummaryByID(alloc.Job.ID) expected = structs.JobSummary{ JobID: alloc.Job.ID, Summary: map[string]structs.TaskGroupSummary{ "web": structs.TaskGroupSummary{ Queued: 10, Starting: 1, }, }, CreateIndex: 1010, ModifyIndex: out2.ModifyIndex, } if !reflect.DeepEqual(&expected, out2) { t.Fatalf("expected: %#v, actual: %#v", &expected, out2) } }
func TestFSM_UpsertAllocs_SharedJob(t *testing.T) { fsm := testFSM(t) alloc := mock.Alloc() fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) job := alloc.Job alloc.Job = nil req := structs.AllocUpdateRequest{ Job: job, Alloc: []*structs.Allocation{alloc}, } buf, err := structs.Encode(structs.AllocUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out, err := fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } alloc.CreateIndex = out.CreateIndex alloc.ModifyIndex = out.ModifyIndex alloc.AllocModifyIndex = out.AllocModifyIndex // Job should be re-attached alloc.Job = job if !reflect.DeepEqual(alloc, out) { t.Fatalf("bad: %#v %#v", alloc, out) } // Ensure that the original job is used evictAlloc := new(structs.Allocation) *evictAlloc = *alloc job = mock.Job() job.Priority = 123 evictAlloc.Job = nil evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict req2 := structs.AllocUpdateRequest{ Job: job, Alloc: []*structs.Allocation{evictAlloc}, } buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) if err != nil { t.Fatalf("err: %v", err) } resp = fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are evicted out, err = fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } if out.DesiredStatus != structs.AllocDesiredStatusEvict { t.Fatalf("alloc found!") } if out.Job == nil || out.Job.Priority == 123 { t.Fatalf("bad job") } }
func TestFSM_UpdateAllocFromClient(t *testing.T) { fsm := testFSM(t) fsm.blockedEvals.SetEnabled(true) state := fsm.State() node := mock.Node() state.UpsertNode(1, node) // Mark an eval as blocked. eval := mock.Eval() eval.ClassEligibility = map[string]bool{node.ComputedClass: true} fsm.blockedEvals.Block(eval) bStats := fsm.blockedEvals.Stats() if bStats.TotalBlocked != 1 { t.Fatalf("bad: %#v", bStats) } // Create a completed eval alloc := mock.Alloc() alloc.NodeID = node.ID state.UpsertAllocs(1, []*structs.Allocation{alloc}) clientAlloc := new(structs.Allocation) *clientAlloc = *alloc clientAlloc.ClientStatus = structs.AllocClientStatusDead req := structs.AllocUpdateRequest{ Alloc: []*structs.Allocation{clientAlloc}, } buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) if err != nil { t.Fatalf("err: %v", err) } resp := fsm.Apply(makeLog(buf)) if resp != nil { t.Fatalf("resp: %v", resp) } // Verify we are registered out, err := fsm.State().AllocByID(alloc.ID) if err != nil { t.Fatalf("err: %v", err) } clientAlloc.CreateIndex = out.CreateIndex clientAlloc.ModifyIndex = out.ModifyIndex if !reflect.DeepEqual(clientAlloc, out) { t.Fatalf("bad: %#v %#v", clientAlloc, out) } // Verify the eval was unblocked. testutil.WaitForResult(func() (bool, error) { bStats = fsm.blockedEvals.Stats() if bStats.TotalBlocked != 0 { return false, fmt.Errorf("bad: %#v %#v", bStats, out) } return true, nil }, func(err error) { t.Fatalf("err: %s", err) }) }