// TestNodeLivenessRestart verifies that if nodes are shutdown and
// restarted, the node liveness records are re-gossiped immediately.
func TestNodeLivenessRestart(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := &multiTestContext{}
defer mtc.Stop()
mtc.Start(t, 2)
// After verifying node is in liveness table, stop store.
verifyLiveness(t, mtc)
mtc.stopStore(0)
// Clear the liveness records in store 1's gossip to make sure we're
// seeing the liveness record properly gossiped at store startup.
var expKeys []string
for _, g := range mtc.gossips {
key := gossip.MakeNodeLivenessKey(g.NodeID.Get())
expKeys = append(expKeys, key)
if err := g.AddInfoProto(key, &storage.Liveness{}, 0); err != nil {
t.Fatal(err)
}
}
sort.Strings(expKeys)
// Register a callback to gossip in order to verify liveness records
// are re-gossiped.
var keysMu struct {
syncutil.Mutex
keys []string
}
livenessRegex := gossip.MakePrefixPattern(gossip.KeyNodeLivenessPrefix)
mtc.gossips[0].RegisterCallback(livenessRegex, func(key string, _ roachpb.Value) {
keysMu.Lock()
defer keysMu.Unlock()
for _, k := range keysMu.keys {
if k == key {
return
}
}
keysMu.keys = append(keysMu.keys, key)
})
// Restart store and verify gossip contains liveness record for nodes 1&2.
mtc.restartStore(0)
testutils.SucceedsSoon(t, func() error {
keysMu.Lock()
defer keysMu.Unlock()
sort.Strings(keysMu.keys)
if !reflect.DeepEqual(keysMu.keys, expKeys) {
return errors.Errorf("expected keys %+v != keys %+v", expKeys, keysMu.keys)
}
return nil
})
}
// TestNodeLivenessSelf verifies that a node keeps its own most
// recent liveness heartbeat info in preference to anything which
// might be received belatedly through gossip.
func TestNodeLivenessSelf(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := &multiTestContext{}
defer mtc.Stop()
mtc.Start(t, 1)
// Verify liveness of all nodes for all nodes.
pauseNodeLivenessHeartbeats(mtc, true)
g := mtc.gossips[0]
liveness, _ := mtc.nodeLivenesses[0].GetLiveness(g.NodeID.Get())
if err := mtc.nodeLivenesses[0].Heartbeat(context.Background(), liveness); err != nil {
t.Fatal(err)
}
// Gossip random nonsense for liveness and verify that asking for
// the node's own node ID returns the "correct" value.
key := gossip.MakeNodeLivenessKey(g.NodeID.Get())
var count int32
g.RegisterCallback(key, func(_ string, val roachpb.Value) {
atomic.AddInt32(&count, 1)
})
testutils.SucceedsSoon(t, func() error {
if err := g.AddInfoProto(key, &storage.Liveness{
NodeID: 1,
Epoch: 2,
}, 0); err != nil {
t.Fatal(err)
}
if atomic.LoadInt32(&count) < 2 {
return errors.New("expected count >= 2")
}
return nil
})
// Self should not see new epoch.
l := mtc.nodeLivenesses[0]
lGet, err := l.GetLiveness(g.NodeID.Get())
if err != nil {
t.Fatal(err)
}
lSelf, err := l.Self()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(lGet, lSelf) {
t.Errorf("expected GetLiveness() to return same value as Self(): %+v != %+v", lGet, lSelf)
}
if lGet.Epoch == 2 || lSelf.NodeID == 2 {
t.Errorf("expected GetLiveness() and Self() not to return artificially gossiped liveness: %+v, %+v", lGet, lSelf)
}
}