// TestRawNodeProposeAndConfChange ensures that RawNode.Propose and RawNode.ProposeConfChange // send the given proposal and ConfChange to the underlying raft. func TestRawNodeProposeAndConfChange(t *testing.T) { s := NewMemoryStorage() var err error rawNode, err := NewRawNode(newTestConfig(1, nil, 10, 1, s), []Peer{{ID: 1}}) if err != nil { t.Fatal(err) } rd := rawNode.Ready() s.Append(rd.Entries) rawNode.Advance(rd) rawNode.Campaign() proposed := false var ( lastIndex uint64 ccdata []byte ) for { rd = rawNode.Ready() s.Append(rd.Entries) // Once we are the leader, propose a command and a ConfChange. if !proposed && rd.SoftState.Lead == rawNode.raft.ID { rawNode.Propose([]byte("somedata")) cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err = cc.Marshal() if err != nil { t.Fatal(err) } rawNode.ProposeConfChange(cc) proposed = true } rawNode.Advance(rd) // Exit when we have four entries: one ConfChange, one no-op for the election, // our proposed command and proposed ConfChange. lastIndex = s.LastIndex() if lastIndex >= 4 { break } } entries, err := s.Entries(lastIndex-1, lastIndex+1, noLimit) if err != nil { t.Fatal(err) } if len(entries) != 2 { t.Fatalf("len(entries) = %d, want %d", len(entries), 2) } if !bytes.Equal(entries[0].Data, []byte("somedata")) { t.Errorf("entries[0].Data = %v, want %v", entries[0].Data, []byte("somedata")) } if entries[1].Type != raftpb.EntryConfChange { t.Fatalf("type = %v, want %v", entries[1].Type, raftpb.EntryConfChange) } if !bytes.Equal(entries[1].Data, ccdata) { t.Errorf("data = %v, want %v", entries[1].Data, ccdata) } }
// StartNode returns a new Node given configuration and a list of raft peers. // It appends a ConfChangeAddNode entry for each given peer to the initial log. func StartNode(c *Config, peers []Peer) Node { r := newRaft(c) // become the follower at term 1 and apply initial configuration // entries of term 1 r.becomeFollower(1, None) for _, peer := range peers { cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} d, err := cc.Marshal() if err != nil { panic("unexpected marshal error") } e := pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: r.raftLog.lastIndex() + 1, Data: d} r.raftLog.append(e) } // Mark these initial entries as committed. // TODO(bdarnell): These entries are still unstable; do we need to preserve // the invariant that committed < unstable? r.raftLog.committed = r.raftLog.lastIndex() // Now apply them, mainly so that the application can call Campaign // immediately after StartNode in tests. Note that these nodes will // be added to raft twice: here and when the application's Ready // loop calls ApplyConfChange. The calls to addNode must come after // all calls to raftLog.append so progress.next is set after these // bootstrapping entries (it is an error if we try to append these // entries since they have already been committed). // We do not set raftLog.applied so the application will be able // to observe all conf changes via Ready.CommittedEntries. for _, peer := range peers { r.addNode(peer.ID) } n := newNode() go n.run(r) return &n }
func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChange) error { data, err := cc.Marshal() if err != nil { return err } return n.Step(ctx, pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange, Data: data}}}) }
func (c *RemoveCommand) Data5() ([]byte, error) { req5 := raftpb.ConfChange{ ID: 0, Type: raftpb.ConfChangeRemoveNode, NodeID: c.id, } return req5.Marshal() }
// TestRawNodeStart ensures that a node can be started correctly. The node should // start with correct configuration change entries, and can accept and commit // proposals. func TestRawNodeStart(t *testing.T) { cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err := cc.Marshal() if err != nil { t.Fatalf("unexpected marshal error: %v", err) } wants := []Ready{ { HardState: raftpb.HardState{Term: 1, Commit: 1, Vote: 0}, Entries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, }, CommittedEntries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, }, }, { HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, }, } storage := NewMemoryStorage() rawNode, err := NewRawNode(newTestConfig(1, nil, 10, 1, storage), []Peer{{ID: 1}}) if err != nil { t.Fatal(err) } rd := rawNode.Ready() t.Logf("rd %v", rd) if !reflect.DeepEqual(rd, wants[0]) { t.Fatalf("#%d: g = %+v,\n w %+v", 1, rd, wants[0]) } else { storage.Append(rd.Entries) rawNode.Advance(rd) } storage.Append(rd.Entries) rawNode.Advance(rd) rawNode.Campaign() rd = rawNode.Ready() storage.Append(rd.Entries) rawNode.Advance(rd) rawNode.Propose([]byte("foo")) if rd = rawNode.Ready(); !reflect.DeepEqual(rd, wants[1]) { t.Errorf("#%d: g = %+v,\n w %+v", 2, rd, wants[1]) } else { storage.Append(rd.Entries) rawNode.Advance(rd) } if rawNode.HasReady() { t.Errorf("unexpected Ready: %+v", rawNode.Ready()) } }
func (n *nodeConfChangeCommitterRecorder) ProposeConfChange(ctx context.Context, conf raftpb.ConfChange) error { data, err := conf.Marshal() if err != nil { return err } n.index++ n.Record(testutil.Action{Name: "ProposeConfChange:" + conf.Type.String()}) n.readyc <- raft.Ready{CommittedEntries: []raftpb.Entry{{Index: n.index, Type: raftpb.EntryConfChange, Data: data}}} return nil }
// TestMultiNodeStart ensures that a node can be started correctly. The node should // start with correct configuration change entries, and can accept and commit // proposals. func TestMultiNodeStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err := cc.Marshal() if err != nil { t.Fatalf("unexpected marshal error: %v", err) } wants := []Ready{ { SoftState: &SoftState{Lead: 1, RaftState: StateLeader}, HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1}, Entries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, {Term: 2, Index: 2}, }, CommittedEntries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, {Term: 2, Index: 2}, }, }, { HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, }, } mn := StartMultiNode(1) storage := NewMemoryStorage() mn.CreateGroup(1, newTestConfig(1, nil, 10, 1, storage), []Peer{{ID: 1}}) mn.Campaign(ctx, 1) gs := <-mn.Ready() g := gs[1] if !reflect.DeepEqual(g, wants[0]) { t.Fatalf("#%d: g = %+v,\n w %+v", 1, g, wants[0]) } else { storage.Append(g.Entries) mn.Advance(gs) } mn.Propose(ctx, 1, []byte("foo")) if gs2 := <-mn.Ready(); !reflect.DeepEqual(gs2[1], wants[1]) { t.Errorf("#%d: g = %+v,\n w %+v", 2, gs2[1], wants[1]) } else { storage.Append(gs2[1].Entries) mn.Advance(gs2) } select { case rd := <-mn.Ready(): t.Errorf("unexpected Ready: %+v", rd) case <-time.After(time.Millisecond): } }
// ProposeConfChange proposes a config change. func (rn *RawNode) ProposeConfChange(cc pb.ConfChange) error { data, err := cc.Marshal() if err != nil { return err } return rn.raft.Step(pb.Message{ Type: pb.MsgProp, Entries: []pb.Entry{ {Type: pb.EntryConfChange, Data: data}, }, }) }
func (mn *multiNode) ProposeConfChange(ctx context.Context, group uint64, cc pb.ConfChange) error { data, err := cc.Marshal() if err != nil { return err } return mn.Step(ctx, group, pb.Message{ Type: pb.MsgProp, Entries: []pb.Entry{ {Type: pb.EntryConfChange, Data: data}, }, }) }
// TestMultiNodeProposeConfig ensures that multiNode.ProposeConfChange // sends the given configuration proposal to the underlying raft. func TestMultiNodeProposeConfig(t *testing.T) { mn := newMultiNode(1) go mn.run() s := NewMemoryStorage() mn.CreateGroup(1, newTestConfig(1, nil, 10, 1, s), []Peer{{ID: 1}}) mn.Campaign(context.TODO(), 1) proposed := false var lastIndex uint64 var ccdata []byte for { rds := <-mn.Ready() rd := rds[1] s.Append(rd.Entries) // change the step function to appendStep until this raft becomes leader if !proposed && rd.SoftState.Lead == mn.id { cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} var err error ccdata, err = cc.Marshal() if err != nil { t.Fatal(err) } mn.ProposeConfChange(context.TODO(), 1, cc) proposed = true } mn.Advance(rds) var err error lastIndex, err = s.LastIndex() if err != nil { t.Fatal(err) } if lastIndex >= 3 { break } } mn.Stop() entries, err := s.Entries(lastIndex, lastIndex+1, noLimit) if err != nil { t.Fatal(err) } if len(entries) != 1 { t.Fatalf("len(entries) = %d, want %d", len(entries), 1) } if entries[0].Type != raftpb.EntryConfChange { t.Fatalf("type = %v, want %v", entries[0].Type, raftpb.EntryConfChange) } if !bytes.Equal(entries[0].Data, ccdata) { t.Errorf("data = %v, want %v", entries[0].Data, ccdata) } }
// makeWAL creates a WAL for the initial cluster func makeWAL(waldir string, cl *membership.RaftCluster) { if err := os.MkdirAll(waldir, 0755); err != nil { ExitWithError(ExitIO, err) } m := cl.MemberByName(restoreName) md := &etcdserverpb.Metadata{NodeID: uint64(m.ID), ClusterID: uint64(cl.ID())} metadata, merr := md.Marshal() if merr != nil { ExitWithError(ExitInvalidInput, merr) } w, walerr := wal.Create(waldir, metadata) if walerr != nil { ExitWithError(ExitIO, walerr) } defer w.Close() peers := make([]raft.Peer, len(cl.MemberIDs())) for i, id := range cl.MemberIDs() { ctx, err := json.Marshal((*cl).Member(id)) if err != nil { ExitWithError(ExitInvalidInput, err) } peers[i] = raft.Peer{ID: uint64(id), Context: ctx} } ents := make([]raftpb.Entry, len(peers)) for i, p := range peers { cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: p.ID, Context: p.Context} d, err := cc.Marshal() if err != nil { ExitWithError(ExitInvalidInput, err) } e := raftpb.Entry{ Type: raftpb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: d, } ents[i] = e } w.Save(raftpb.HardState{ Term: 1, Vote: peers[0].ID, Commit: uint64(len(ents))}, ents) }
func TestNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err := cc.Marshal() if err != nil { t.Fatalf("unexpected marshal error: %v", err) } wants := []Ready{ { SoftState: &SoftState{Lead: 1, Nodes: []uint64{1}, RaftState: StateLeader}, HardState: raftpb.HardState{Term: 1, Commit: 2}, Entries: []raftpb.Entry{ {}, {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, {Term: 1, Index: 2}, }, CommittedEntries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, {Term: 1, Index: 2}, }, }, { HardState: raftpb.HardState{Term: 1, Commit: 3}, Entries: []raftpb.Entry{{Term: 1, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 1, Index: 3, Data: []byte("foo")}}, }, } n := StartNode(1, []Peer{{ID: 1}}, 10, 1) n.ApplyConfChange(cc) n.Campaign(ctx) if g := <-n.Ready(); !reflect.DeepEqual(g, wants[0]) { t.Errorf("#%d: g = %+v,\n w %+v", 1, g, wants[0]) } n.Propose(ctx, []byte("foo")) if g := <-n.Ready(); !reflect.DeepEqual(g, wants[1]) { t.Errorf("#%d: g = %+v,\n w %+v", 2, g, wants[1]) } select { case rd := <-n.Ready(): t.Errorf("unexpected Ready: %+v", rd) default: } }
// NewRawNode returns a new RawNode given configuration and a list of raft peers. func NewRawNode(config *Config, peers []Peer) (*RawNode, error) { if config.ID == 0 { panic("config.ID must not be zero") } r := newRaft(config) rn := &RawNode{ raft: r, } lastIndex, err := config.Storage.LastIndex() if err != nil { panic(err) // TODO(bdarnell) } // If the log is empty, this is a new RawNode (like StartNode); otherwise it's // restoring an existing RawNode (like RestartNode). // TODO(bdarnell): rethink RawNode initialization and whether the application needs // to be able to tell us when it expects the RawNode to exist. if lastIndex == 0 { r.becomeFollower(1, None) ents := make([]pb.Entry, len(peers)) for i, peer := range peers { cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} data, err := cc.Marshal() if err != nil { panic("unexpected marshal error") } ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data} } r.raftLog.append(ents...) r.raftLog.committed = uint64(len(ents)) for _, peer := range peers { r.addNode(peer.ID) } } // Set the initial hard and soft states after performing all initialization. rn.prevSoftSt = r.softState() if lastIndex == 0 { rn.prevHardSt = emptyState } else { rn.prevHardSt = r.hardState() } return rn, nil }
// StartNode returns a new Node given a unique raft id, a list of raft peers, and // the election and heartbeat timeouts in units of ticks. // It also builds ConfChangeAddNode entry for each peer and puts them at the head of the log. func StartNode(id uint64, peers []Peer, election, heartbeat int) Node { n := newNode() r := newRaft(id, nil, election, heartbeat) for _, peer := range peers { cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} d, err := cc.Marshal() if err != nil { panic("unexpected marshal error") } e := pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: r.raftLog.lastIndex() + 1, Data: d} r.raftLog.append(r.raftLog.lastIndex(), e) } r.raftLog.committed = r.raftLog.lastIndex() go n.run(r) return &n }
// StartNode returns a new Node given a unique raft id, a list of raft peers, and // the election and heartbeat timeouts in units of ticks. // It appends a ConfChangeAddNode entry for each given peer to the initial log. func StartNode(id uint64, peers []Peer, election, heartbeat int, storage Storage) Node { n := newNode() r := newRaft(id, nil, election, heartbeat, storage) for _, peer := range peers { cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} d, err := cc.Marshal() if err != nil { panic("unexpected marshal error") } e := pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: r.raftLog.lastIndex() + 1, Data: d} r.raftLog.append(r.raftLog.lastIndex(), e) } // Mark these initial entries as committed. // TODO(bdarnell): These entries are still unstable; do we need to preserve // the invariant that committed < unstable? r.raftLog.committed = r.raftLog.lastIndex() go n.run(r) return &n }
// TestNodeProposeConfig ensures that node.ProposeConfChange sends the given configuration proposal // to the underlying raft. func TestNodeProposeConfig(t *testing.T) { msgs := []raftpb.Message{} appendStep := func(r *raft, m raftpb.Message) { msgs = append(msgs, m) } n := newNode() s := NewMemoryStorage() r := newTestRaft(1, []uint64{1}, 10, 1, s) go n.run(r) n.Campaign(context.TODO()) for { rd := <-n.Ready() s.Append(rd.Entries) // change the step function to appendStep until this raft becomes leader if rd.SoftState.Lead == r.id { r.step = appendStep n.Advance() break } n.Advance() } cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err := cc.Marshal() if err != nil { t.Fatal(err) } n.ProposeConfChange(context.TODO(), cc) n.Stop() if len(msgs) != 1 { t.Fatalf("len(msgs) = %d, want %d", len(msgs), 1) } if msgs[0].Type != raftpb.MsgProp { t.Errorf("msg type = %d, want %d", msgs[0].Type, raftpb.MsgProp) } if !bytes.Equal(msgs[0].Entries[0].Data, ccdata) { t.Errorf("data = %v, want %v", msgs[0].Entries[0].Data, ccdata) } }
// TestNodeProposeAddDuplicateNode ensures that two proposes to add the same node should // not affect the later propose to add new node. func TestNodeProposeAddDuplicateNode(t *testing.T) { n := newNode() s := NewMemoryStorage() r := newTestRaft(1, []uint64{1}, 10, 1, s) go n.run(r) n.Campaign(context.TODO()) rdyEntries := make([]raftpb.Entry, 0) ticker := time.NewTicker(time.Millisecond * 100) done := make(chan struct{}) stop := make(chan struct{}) applyConfChan := make(chan struct{}) go func() { defer close(done) for { select { case <-stop: return case <-ticker.C: n.Tick() case rd := <-n.Ready(): s.Append(rd.Entries) for _, e := range rd.Entries { rdyEntries = append(rdyEntries, e) switch e.Type { case raftpb.EntryNormal: case raftpb.EntryConfChange: var cc raftpb.ConfChange cc.Unmarshal(e.Data) n.ApplyConfChange(cc) applyConfChan <- struct{}{} } } n.Advance() } } }() cc1 := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata1, _ := cc1.Marshal() n.ProposeConfChange(context.TODO(), cc1) <-applyConfChan // try add the same node again n.ProposeConfChange(context.TODO(), cc1) <-applyConfChan // the new node join should be ok cc2 := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 2} ccdata2, _ := cc2.Marshal() n.ProposeConfChange(context.TODO(), cc2) <-applyConfChan close(stop) <-done if len(rdyEntries) != 4 { t.Errorf("len(entry) = %d, want %d, %v\n", len(rdyEntries), 3, rdyEntries) } if !bytes.Equal(rdyEntries[1].Data, ccdata1) { t.Errorf("data = %v, want %v", rdyEntries[1].Data, ccdata1) } if !bytes.Equal(rdyEntries[3].Data, ccdata2) { t.Errorf("data = %v, want %v", rdyEntries[3].Data, ccdata2) } n.Stop() }
// TestNodeStart ensures that a node can be started correctly. The node should // start with correct configuration change entries, and can accept and commit // proposals. func TestNodeStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err := cc.Marshal() if err != nil { t.Fatalf("unexpected marshal error: %v", err) } wants := []Ready{ { HardState: raftpb.HardState{Term: 1, Commit: 1, Vote: 0}, Entries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, }, CommittedEntries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, }, }, { HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, }, } storage := NewMemoryStorage() c := &Config{ ID: 1, ElectionTick: 10, HeartbeatTick: 1, Storage: storage, MaxSizePerMsg: noLimit, MaxInflightMsgs: 256, } n := StartNode(c, []Peer{{ID: 1}}) defer n.Stop() g := <-n.Ready() if !reflect.DeepEqual(g, wants[0]) { t.Fatalf("#%d: g = %+v,\n w %+v", 1, g, wants[0]) } else { storage.Append(g.Entries) n.Advance() } n.Campaign(ctx) rd := <-n.Ready() storage.Append(rd.Entries) n.Advance() n.Propose(ctx, []byte("foo")) if g2 := <-n.Ready(); !reflect.DeepEqual(g2, wants[1]) { t.Errorf("#%d: g = %+v,\n w %+v", 2, g2, wants[1]) } else { storage.Append(g2.Entries) n.Advance() } select { case rd := <-n.Ready(): t.Errorf("unexpected Ready: %+v", rd) case <-time.After(time.Millisecond): } }
// makeWAL creates a WAL for the initial cluster func makeWALAndSnap(waldir, snapdir string, cl *membership.RaftCluster) { if err := fileutil.CreateDirAll(waldir); err != nil { ExitWithError(ExitIO, err) } // add members again to persist them to the store we create. st := store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) cl.SetStore(st) for _, m := range cl.Members() { cl.AddMember(m) } m := cl.MemberByName(restoreName) md := &etcdserverpb.Metadata{NodeID: uint64(m.ID), ClusterID: uint64(cl.ID())} metadata, merr := md.Marshal() if merr != nil { ExitWithError(ExitInvalidInput, merr) } w, walerr := wal.Create(waldir, metadata) if walerr != nil { ExitWithError(ExitIO, walerr) } defer w.Close() peers := make([]raft.Peer, len(cl.MemberIDs())) for i, id := range cl.MemberIDs() { ctx, err := json.Marshal((*cl).Member(id)) if err != nil { ExitWithError(ExitInvalidInput, err) } peers[i] = raft.Peer{ID: uint64(id), Context: ctx} } ents := make([]raftpb.Entry, len(peers)) nodeIDs := make([]uint64, len(peers)) for i, p := range peers { nodeIDs[i] = p.ID cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: p.ID, Context: p.Context} d, err := cc.Marshal() if err != nil { ExitWithError(ExitInvalidInput, err) } e := raftpb.Entry{ Type: raftpb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: d, } ents[i] = e } commit, term := uint64(len(ents)), uint64(1) if err := w.Save(raftpb.HardState{ Term: term, Vote: peers[0].ID, Commit: commit}, ents); err != nil { ExitWithError(ExitIO, err) } b, berr := st.Save() if berr != nil { ExitWithError(ExitError, berr) } raftSnap := raftpb.Snapshot{ Data: b, Metadata: raftpb.SnapshotMetadata{ Index: commit, Term: term, ConfState: raftpb.ConfState{ Nodes: nodeIDs, }, }, } snapshotter := snap.New(snapdir) if err := snapshotter.SaveSnap(raftSnap); err != nil { panic(err) } if err := w.SaveSnapshot(walpb.Snapshot{Index: commit, Term: term}); err != nil { ExitWithError(ExitIO, err) } }
func (mn *multiNode) run() { groups := map[uint64]*groupState{} rds := map[uint64]Ready{} var advancec chan map[uint64]Ready for { // Only select readyc if we have something to report and we are not // currently waiting for an advance. readyc := mn.readyc if len(rds) == 0 || advancec != nil { readyc = nil } // group points to the group that was touched on this iteration (if any) var group *groupState select { case gc := <-mn.groupc: gc.config.ID = mn.id r := newRaft(gc.config) group = &groupState{ id: gc.id, raft: r, } groups[gc.id] = group lastIndex, err := gc.config.Storage.LastIndex() if err != nil { panic(err) // TODO(bdarnell) } // If the log is empty, this is a new group (like StartNode); otherwise it's // restoring an existing group (like RestartNode). // TODO(bdarnell): rethink group initialization and whether the application needs // to be able to tell us when it expects the group to exist. if lastIndex == 0 { r.becomeFollower(1, None) ents := make([]pb.Entry, len(gc.peers)) for i, peer := range gc.peers { cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} data, err := cc.Marshal() if err != nil { panic("unexpected marshal error") } ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data} } r.raftLog.append(ents...) r.raftLog.committed = uint64(len(ents)) for _, peer := range gc.peers { r.addNode(peer.ID) } } // Set the initial hard and soft states after performing all initialization. group.prevSoftSt = r.softState() group.prevHardSt = r.HardState close(gc.done) case gr := <-mn.rmgroupc: delete(groups, gr.id) delete(rds, gr.id) close(gr.done) case mm := <-mn.propc: // TODO(bdarnell): single-node impl doesn't read from propc unless the group // has a leader; we can't do that since we have one propc for many groups. // We'll have to buffer somewhere on a group-by-group basis, or just let // raft.Step drop any such proposals on the floor. mm.msg.From = mn.id var ok bool if group, ok = groups[mm.group]; ok { group.raft.Step(mm.msg) } case mm := <-mn.recvc: group = groups[mm.group] if _, ok := group.raft.prs[mm.msg.From]; ok || !IsResponseMsg(mm.msg) { group.raft.Step(mm.msg) } case mcc := <-mn.confc: group = groups[mcc.group] if mcc.msg.NodeID == None { group.raft.resetPendingConf() select { case mcc.ch <- pb.ConfState{Nodes: group.raft.nodes()}: case <-mn.done: } break } switch mcc.msg.Type { case pb.ConfChangeAddNode: group.raft.addNode(mcc.msg.NodeID) case pb.ConfChangeRemoveNode: group.raft.removeNode(mcc.msg.NodeID) case pb.ConfChangeUpdateNode: group.raft.resetPendingConf() default: panic("unexpected conf type") } select { case mcc.ch <- pb.ConfState{Nodes: group.raft.nodes()}: case <-mn.done: } case <-mn.tickc: // TODO(bdarnell): instead of calling every group on every tick, // we should have a priority queue of groups based on their next // time-based event. for _, g := range groups { g.raft.tick() rd := g.newReady() if rd.containsUpdates() { rds[g.id] = rd } } case readyc <- rds: // Clear outgoing messages as soon as we've passed them to the application. for g := range rds { groups[g].raft.msgs = nil } rds = map[uint64]Ready{} advancec = mn.advancec case advs := <-advancec: for groupID, rd := range advs { g, ok := groups[groupID] if !ok { continue } g.commitReady(rd) // We've been accumulating new entries in rds which may now be obsolete. // Drop the old Ready object and create a new one if needed. delete(rds, groupID) newRd := g.newReady() if newRd.containsUpdates() { rds[groupID] = newRd } } advancec = nil case ms := <-mn.status: if g, ok := groups[ms.group]; ok { s := getStatus(g.raft) ms.ch <- &s } else { ms.ch <- nil } case <-mn.stop: close(mn.done) return } if group != nil { rd := group.newReady() if rd.containsUpdates() { rds[group.id] = rd } } } }