func TestRaftRejoin(t *testing.T) { t.Parallel() nodes, clockSource := raftutils.NewRaftCluster(t, tc) defer raftutils.TeardownCluster(t, nodes) ids := []string{"id1", "id2"} // Propose a value values := make([]*api.Node, 2) var err error values[0], err = raftutils.ProposeValue(t, nodes[1], ids[0]) assert.NoError(t, err, "failed to propose value") // The value should be replicated on node 3 raftutils.CheckValue(t, clockSource, nodes[3], values[0]) assert.Equal(t, len(nodes[3].GetMemberlist()), 3) // Stop node 3 nodes[3].Server.Stop() nodes[3].Shutdown() // Propose another value values[1], err = raftutils.ProposeValue(t, nodes[1], ids[1]) assert.NoError(t, err, "failed to propose value") // Nodes 1 and 2 should have the new value raftutils.CheckValuesOnNodes(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2]}, ids, values) nodes[3] = raftutils.RestartNode(t, clockSource, nodes[3], false) raftutils.WaitForCluster(t, clockSource, nodes) // Node 3 should have all values, including the one proposed while // it was unavailable. raftutils.CheckValuesOnNodes(t, clockSource, nodes, ids, values) }
func TestStress(t *testing.T) { t.Parallel() // Bring up a 5 nodes cluster nodes, clockSource := raftutils.NewRaftCluster(t, tc) raftutils.AddRaftNode(t, clockSource, nodes, tc) raftutils.AddRaftNode(t, clockSource, nodes, tc) defer raftutils.TeardownCluster(t, nodes) // number of nodes that are running nup := len(nodes) // record of nodes that are down idleNodes := map[int]struct{}{} // record of ids that proposed successfully or time-out pIDs := []string{} leader := -1 for iters := 0; iters < 1000; iters++ { // keep proposing new values and killing leader for i := 1; i <= 5; i++ { if nodes[uint64(i)] != nil { id := strconv.Itoa(iters) _, err := raftutils.ProposeValue(t, nodes[uint64(i)], id) if err == nil { pIDs = append(pIDs, id) // if propose successfully, at least there are 3 running nodes assert.True(t, nup >= 3) // only leader can propose value assert.True(t, leader == i || leader == -1) // update leader leader = i break } else if strings.Contains(err.Error(), "context deadline exceeded") { // though it's timing out, we still record this value // for it may be proposed successfully and stored in Raft some time later pIDs = append(pIDs, id) } } } if rand.Intn(100) < 10 { // increase clock to make potential election finish quickly clockSource.Increment(200 * time.Millisecond) time.Sleep(10 * time.Millisecond) } else { ms := rand.Intn(10) clockSource.Increment(time.Duration(ms) * time.Millisecond) } if leader != -1 { // if propose successfully, try to kill a node in random s := rand.Intn(5) + 1 if _, ok := idleNodes[s]; !ok { id := uint64(s) nodes[id].Server.Stop() nodes[id].Shutdown() idleNodes[s] = struct{}{} nup -= 1 if s == leader { // leader is killed leader = -1 } } } if nup < 3 { // if quorum is lost, try to bring back a node s := rand.Intn(5) + 1 if _, ok := idleNodes[s]; ok { id := uint64(s) nodes[id] = raftutils.RestartNode(t, clockSource, nodes[id], false) delete(idleNodes, s) nup++ } } } // bring back all nodes and propose the final value for i := range idleNodes { id := uint64(i) nodes[id] = raftutils.RestartNode(t, clockSource, nodes[id], false) } raftutils.WaitForCluster(t, clockSource, nodes) id := strconv.Itoa(1000) val, err := raftutils.ProposeValue(t, raftutils.Leader(nodes), id) assert.NoError(t, err, "failed to propose value") pIDs = append(pIDs, id) // increase clock to make cluster stable time.Sleep(500 * time.Millisecond) clockSource.Increment(500 * time.Millisecond) ids, values := raftutils.GetAllValuesOnNode(t, clockSource, nodes[1]) // since cluster is stable, final value must be in the raft store find := false for _, value := range values { if reflect.DeepEqual(value, val) { find = true break } } assert.True(t, find) // all nodes must have the same value raftutils.CheckValuesOnNodes(t, clockSource, nodes, ids, values) // ids should be a subset of pIDs for _, id := range ids { find = false for _, pid := range pIDs { if id == pid { find = true break } } assert.True(t, find) } }
func TestRaftSnapshotRestart(t *testing.T) { t.Parallel() // Bring up a 3 node cluster nodes, clockSource := raftutils.NewRaftCluster(t, tc, &api.RaftConfig{SnapshotInterval: 10, LogEntriesForSlowFollowers: 0}) defer raftutils.TeardownCluster(t, nodes) nodeIDs := []string{"id1", "id2", "id3", "id4", "id5", "id6", "id7"} values := make([]*api.Node, len(nodeIDs)) // Propose 3 values var err error for i, nodeID := range nodeIDs[:3] { values[i], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeID) assert.NoError(t, err, "failed to propose value") } // Take down node 3 nodes[3].Server.Stop() nodes[3].Shutdown() // Propose a 4th value before the snapshot values[3], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[3]) assert.NoError(t, err, "failed to propose value") // Remaining nodes shouldn't have snapshot files yet for _, node := range []*raftutils.TestNode{nodes[1], nodes[2]} { dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap")) assert.NoError(t, err) assert.Len(t, dirents, 0) } // Add a node to the cluster before the snapshot. This is the event // that triggers the snapshot. nodes[4] = raftutils.NewJoinNode(t, clockSource, nodes[1].Address, tc) raftutils.WaitForCluster(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2], 4: nodes[4]}) // Remaining nodes should now have a snapshot file for nodeIdx, node := range []*raftutils.TestNode{nodes[1], nodes[2]} { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap")) if err != nil { return err } if len(dirents) != 1 { return fmt.Errorf("expected 1 snapshot, found %d on node %d", len(dirents), nodeIdx+1) } return nil })) } raftutils.CheckValuesOnNodes(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2]}, nodeIDs[:4], values[:4]) // Propose a 5th value values[4], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[4]) require.NoError(t, err) // Add another node to the cluster nodes[5] = raftutils.NewJoinNode(t, clockSource, nodes[1].Address, tc) raftutils.WaitForCluster(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2], 4: nodes[4], 5: nodes[5]}) // New node should get a copy of the snapshot assert.NoError(t, raftutils.PollFunc(clockSource, func() error { dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap")) if err != nil { return err } if len(dirents) != 1 { return fmt.Errorf("expected 1 snapshot, found %d on new node", len(dirents)) } return nil })) dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap")) assert.NoError(t, err) assert.Len(t, dirents, 1) raftutils.CheckValuesOnNodes(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2]}, nodeIDs[:5], values[:5]) // It should know about the other nodes, including the one that was just added stripMembers := func(memberList map[uint64]*api.RaftMember) map[uint64]*api.RaftMember { raftNodes := make(map[uint64]*api.RaftMember) for k, v := range memberList { raftNodes[k] = &api.RaftMember{ RaftID: v.RaftID, Addr: v.Addr, } } return raftNodes } assert.Equal(t, stripMembers(nodes[1].GetMemberlist()), stripMembers(nodes[4].GetMemberlist())) // Restart node 3 nodes[3] = raftutils.RestartNode(t, clockSource, nodes[3], false) raftutils.WaitForCluster(t, clockSource, nodes) // Node 3 should know about other nodes, including the new one assert.Len(t, nodes[3].GetMemberlist(), 5) assert.Equal(t, stripMembers(nodes[1].GetMemberlist()), stripMembers(nodes[3].GetMemberlist())) // Propose yet another value, to make sure the rejoined node is still // receiving new logs values[5], err = raftutils.ProposeValue(t, raftutils.Leader(nodes), DefaultProposalTime, nodeIDs[5]) require.NoError(t, err) // All nodes should have all the data raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:6], values[:6]) // Restart node 3 again. It should load the snapshot. nodes[3].Server.Stop() nodes[3].Shutdown() nodes[3] = raftutils.RestartNode(t, clockSource, nodes[3], false) raftutils.WaitForCluster(t, clockSource, nodes) assert.Len(t, nodes[3].GetMemberlist(), 5) assert.Equal(t, stripMembers(nodes[1].GetMemberlist()), stripMembers(nodes[3].GetMemberlist())) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:6], values[:6]) // Propose again. Just to check consensus after this latest restart. values[6], err = raftutils.ProposeValue(t, raftutils.Leader(nodes), DefaultProposalTime, nodeIDs[6]) require.NoError(t, err) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) }
func TestRaftSnapshot(t *testing.T) { t.Parallel() // Bring up a 3 node cluster nodes, clockSource := raftutils.NewRaftCluster(t, tc, &api.RaftConfig{SnapshotInterval: 9, LogEntriesForSlowFollowers: 0}) defer raftutils.TeardownCluster(t, nodes) nodeIDs := []string{"id1", "id2", "id3", "id4", "id5", "id6", "id7", "id8", "id9", "id10", "id11", "id12"} values := make([]*api.Node, len(nodeIDs)) snapshotFilenames := make(map[uint64]string, 4) // Propose 3 values var err error for i, nodeID := range nodeIDs[:3] { values[i], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeID) assert.NoError(t, err, "failed to propose value") } // None of the nodes should have snapshot files yet for _, node := range nodes { dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap")) assert.NoError(t, err) assert.Len(t, dirents, 0) } // Check all nodes have all the data. // This also acts as a synchronization point so that the next value we // propose will arrive as a separate message to the raft state machine, // and it is guaranteed to have the right cluster settings when // deciding whether to create a new snapshot. raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:3], values) // Propose a 4th value values[3], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[3]) assert.NoError(t, err, "failed to propose value") // All nodes should now have a snapshot file for nodeID, node := range nodes { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap")) if err != nil { return err } if len(dirents) != 1 { return fmt.Errorf("expected 1 snapshot, found %d", len(dirents)) } snapshotFilenames[nodeID] = dirents[0].Name() return nil })) } // Add a node to the cluster raftutils.AddRaftNode(t, clockSource, nodes, tc) // It should get a copy of the snapshot assert.NoError(t, raftutils.PollFunc(clockSource, func() error { dirents, err := ioutil.ReadDir(filepath.Join(nodes[4].StateDir, "snap")) if err != nil { return err } if len(dirents) != 1 { return fmt.Errorf("expected 1 snapshot, found %d on new node", len(dirents)) } snapshotFilenames[4] = dirents[0].Name() return nil })) // It should know about the other nodes stripMembers := func(memberList map[uint64]*api.RaftMember) map[uint64]*api.RaftMember { raftNodes := make(map[uint64]*api.RaftMember) for k, v := range memberList { raftNodes[k] = &api.RaftMember{ RaftID: v.RaftID, Addr: v.Addr, } } return raftNodes } assert.Equal(t, stripMembers(nodes[1].GetMemberlist()), stripMembers(nodes[4].GetMemberlist())) // All nodes should have all the data raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:4], values) // Propose more values to provoke a second snapshot for i := 4; i != len(nodeIDs); i++ { values[i], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[i]) assert.NoError(t, err, "failed to propose value") } // All nodes should have a snapshot under a *different* name for nodeID, node := range nodes { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap")) if err != nil { return err } if len(dirents) != 1 { return fmt.Errorf("expected 1 snapshot, found %d on node %d", len(dirents), nodeID) } if dirents[0].Name() == snapshotFilenames[nodeID] { return fmt.Errorf("snapshot %s did not get replaced", snapshotFilenames[nodeID]) } return nil })) } // All nodes should have all the data raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) }
// This test rotates the encryption key and waits for the expected thing to happen func TestRaftEncryptionKeyRotationWait(t *testing.T) { t.Parallel() nodes := make(map[uint64]*raftutils.TestNode) var clockSource *fakeclock.FakeClock raftConfig := raft.DefaultRaftConfig() nodes[1], clockSource = raftutils.NewInitNode(t, tc, &raftConfig) defer raftutils.TeardownCluster(t, nodes) nodeIDs := []string{"id1", "id2", "id3"} values := make([]*api.Node, len(nodeIDs)) // Propose 3 values var err error for i, nodeID := range nodeIDs[:3] { values[i], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeID) require.NoError(t, err, "failed to propose value") } snapDir := filepath.Join(nodes[1].StateDir, "snap-v3-encrypted") startingKeys := nodes[1].KeyRotator.GetKeys() // rotate the encryption key nodes[1].KeyRotator.QueuePendingKey([]byte("key2")) nodes[1].KeyRotator.RotationNotify() <- struct{}{} // the rotation should trigger a snapshot, which should notify the rotator when it's done require.NoError(t, raftutils.PollFunc(clockSource, func() error { snapshots, err := storage.ListSnapshots(snapDir) if err != nil { return err } if len(snapshots) != 1 { return fmt.Errorf("expected 1 snapshot, found %d on new node", len(snapshots)) } if nodes[1].KeyRotator.NeedsRotation() { return fmt.Errorf("rotation never finished") } return nil })) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) // Propose a 4th value nodeIDs = append(nodeIDs, "id4") v, err := raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, "id4") require.NoError(t, err, "failed to propose value") values = append(values, v) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) nodes[1].Server.Stop() nodes[1].ShutdownRaft() // Try to restart node 1. Without the new unlock key, it can't actually start n, ctx := raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(startingKeys)) require.Error(t, n.Node.JoinAndStart(ctx), "should not have been able to restart since we can't read snapshots") // with the right key, it can start, even if the right key is only the pending key newKeys := startingKeys newKeys.PendingDEK = []byte("key2") nodes[1].KeyRotator = raftutils.NewSimpleKeyRotator(newKeys) nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false) raftutils.WaitForCluster(t, clockSource, nodes) // as soon as we joined, it should have finished rotating the key require.False(t, nodes[1].KeyRotator.NeedsRotation()) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) // break snapshotting, and ensure that key rotation never finishes tempSnapDir := filepath.Join(nodes[1].StateDir, "snap-backup") require.NoError(t, os.Rename(snapDir, tempSnapDir)) require.NoError(t, ioutil.WriteFile(snapDir, []byte("this is no longer a directory"), 0644)) nodes[1].KeyRotator.QueuePendingKey([]byte("key3")) nodes[1].KeyRotator.RotationNotify() <- struct{}{} time.Sleep(250 * time.Millisecond) // rotation has not been finished, because we cannot take a snapshot require.True(t, nodes[1].KeyRotator.NeedsRotation()) // Propose a 5th value, so we have WALs written with the new key nodeIDs = append(nodeIDs, "id5") v, err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, "id5") require.NoError(t, err, "failed to propose value") values = append(values, v) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) nodes[1].Server.Stop() nodes[1].ShutdownRaft() // restore the snapshot dir require.NoError(t, os.RemoveAll(snapDir)) require.NoError(t, os.Rename(tempSnapDir, snapDir)) // Now the wals are a mix of key2 and key3 - we can't actually start with either key singleKey := raft.EncryptionKeys{CurrentDEK: []byte("key2")} n, ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(singleKey)) require.Error(t, n.Node.JoinAndStart(ctx), "should not have been able to restart since we can't read all the WALs, even if we can read the snapshot") singleKey = raft.EncryptionKeys{CurrentDEK: []byte("key3")} n, ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(singleKey)) require.Error(t, n.Node.JoinAndStart(ctx), "should not have been able to restart since we can't read all the WALs, and also not the snapshot") nodes[1], ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(raft.EncryptionKeys{ CurrentDEK: []byte("key2"), PendingDEK: []byte("key3"), })) require.NoError(t, nodes[1].Node.JoinAndStart(ctx)) // we can load, but we still need a snapshot because rotation hasn't finished snapshots, err := storage.ListSnapshots(snapDir) require.NoError(t, err) require.Len(t, snapshots, 1, "expected 1 snapshot") require.True(t, nodes[1].KeyRotator.NeedsRotation()) currSnapshot := snapshots[0] // start the node - everything should fix itself go nodes[1].Node.Run(ctx) raftutils.WaitForCluster(t, clockSource, nodes) require.NoError(t, raftutils.PollFunc(clockSource, func() error { snapshots, err := storage.ListSnapshots(snapDir) if err != nil { return err } if len(snapshots) != 1 { return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) } if snapshots[0] == currSnapshot { return fmt.Errorf("new snapshot not done yet") } if nodes[1].KeyRotator.NeedsRotation() { return fmt.Errorf("rotation never finished") } currSnapshot = snapshots[0] return nil })) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) // If we can't update the keys, we wait for the next snapshot to do so nodes[1].KeyRotator.SetUpdateFunc(func() error { return fmt.Errorf("nope!") }) nodes[1].KeyRotator.QueuePendingKey([]byte("key4")) nodes[1].KeyRotator.RotationNotify() <- struct{}{} require.NoError(t, raftutils.PollFunc(clockSource, func() error { snapshots, err := storage.ListSnapshots(snapDir) if err != nil { return err } if len(snapshots) != 1 { return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) } if snapshots[0] == currSnapshot { return fmt.Errorf("new snapshot not done yet") } currSnapshot = snapshots[0] return nil })) require.True(t, nodes[1].KeyRotator.NeedsRotation()) // Fix updating the key rotator, and propose a 6th value - this should trigger the key // rotation to finish nodes[1].KeyRotator.SetUpdateFunc(nil) nodeIDs = append(nodeIDs, "id6") v, err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, "id6") require.NoError(t, err, "failed to propose value") values = append(values, v) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) require.NoError(t, raftutils.PollFunc(clockSource, func() error { if nodes[1].KeyRotator.NeedsRotation() { return fmt.Errorf("rotation never finished") } return nil })) // no new snapshot snapshots, err = storage.ListSnapshots(snapDir) require.NoError(t, err) require.Len(t, snapshots, 1) require.Equal(t, currSnapshot, snapshots[0]) // Even if something goes wrong with getting keys, and needs rotation returns a false positive, // if there's no PendingDEK nothing happens. fakeTrue := true nodes[1].KeyRotator.SetNeedsRotation(&fakeTrue) nodes[1].KeyRotator.RotationNotify() <- struct{}{} // propose another value nodeIDs = append(nodeIDs, "id7") v, err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, "id7") require.NoError(t, err, "failed to propose value") values = append(values, v) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) // no new snapshot snapshots, err = storage.ListSnapshots(snapDir) require.NoError(t, err) require.Len(t, snapshots, 1) require.Equal(t, currSnapshot, snapshots[0]) // and when we restart, we can restart with the original key (the WAL written for the new proposed value) // is written with the old key nodes[1].Server.Stop() nodes[1].ShutdownRaft() nodes[1].KeyRotator = raftutils.NewSimpleKeyRotator(raft.EncryptionKeys{ CurrentDEK: []byte("key4"), }) nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false) raftutils.WaitForCluster(t, clockSource, nodes) raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs, values) }