func report(err error, batch *raft.Batch, r raft.Reporter) { for g, msgs := range batch.Messages { if unreachable(err) { r.ReportUnreachable(batch.To, g) } for _, msg := range msgs { if !etcdraft.IsEmptySnap(msg.Snapshot) { r.ReportSnapshot(batch.To, g, snapStatus(err)) } } } }
func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error { if raft.IsEmptySnap(snapshot) { return nil } return s.save(&snapshot) }
func (n *MultiNode) handleReadies(readies map[uint64]etcdraft.Ready) { glog.V(3).Infof("%v handles a ready", n) beatBatch := make(nodeBatch) normBatch := make(nodeBatch) snapBatch := make(nodeBatch) saved := make(chan struct{}, len(readies)) for gid, rd := range readies { if !shouldSave(rd) { saved <- struct{}{} continue } g, ok := n.groups[gid] if !ok { glog.Fatalf("cannot find group %v", g) } g.savec <- readySaved{ ready: rd, saved: saved, } } for gid, rd := range readies { for _, m := range rd.Messages { if m.Type == raftpb.MsgHeartbeat || m.Type == raftpb.MsgHeartbeatResp { batch := beatBatch.batch(m.To) // Only one heartbeat/response message should suffice. batch.Messages[gid] = []raftpb.Message{m} continue } var batch *Batch if !etcdraft.IsEmptySnap(m.Snapshot) { batch = snapBatch.batch(m.To) } else { batch = normBatch.batch(m.To) } batch.Messages[gid] = append(batch.Messages[gid], m) } } for i := 0; i < len(readies); i++ { select { case <-saved: case <-n.done: return } } glog.V(3).Infof("%v saved the readies for all groups", n) for nid, batch := range beatBatch { glog.V(3).Infof("%v sends high priority batch to %v", n, nid) batch.From = n.id batch.To = nid batch.Priority = High n.send(batch, n.node) } for nid, batch := range normBatch { glog.V(3).Infof("%v sends normal priority batch to %v", n, nid) batch.From = n.id batch.To = nid batch.Priority = Normal n.send(batch, n.node) } for nid, batch := range snapBatch { glog.V(3).Infof("%v sends low priority batch to %v", n, nid) batch.From = n.id batch.To = nid batch.Priority = Low n.send(batch, n.node) } select { case n.advancec <- readies: case <-n.done: } }
func shouldSave(rd etcdraft.Ready) bool { return rd.SoftState != nil || !etcdraft.IsEmptyHardState(rd.HardState) || !etcdraft.IsEmptySnap(rd.Snapshot) || len(rd.Entries) > 0 || len(rd.CommittedEntries) > 0 }
func (g *group) apply(ready etcdraft.Ready) error { if ready.SoftState != nil { newLead := ready.SoftState.Lead if g.leader != newLead { g.stateMachine.ProcessStatusChange(LeaderChanged{ Old: g.leader, New: newLead, Term: ready.HardState.Term, }) g.leader = newLead } } // Recover from snapshot if it is more recent than the currently // applied. if !etcdraft.IsEmptySnap(ready.Snapshot) && ready.Snapshot.Metadata.Index > g.applied { if err := g.stateMachine.Restore(ready.Snapshot.Data); err != nil { glog.Fatalf("error in recovering the state machine: %v", err) } // FIXME(soheil): update the nodes and notify the application? g.applied = ready.Snapshot.Metadata.Index glog.Infof("%v recovered from incoming snapshot at index %d", g.node, g.snapped) } es := ready.CommittedEntries if len(es) == 0 { return nil } firsti := es[0].Index if firsti > g.applied+1 { glog.Fatalf( "1st index of committed entry[%d] should <= applied[%d] + 1", firsti, g.applied) } if glog.V(3) { glog.Infof("%v receives raft update: committed=%s appended=%s", g, formatEntries(es), formatEntries(ready.Entries)) } for _, e := range es { if e.Index <= g.applied { continue } switch e.Type { case raftpb.EntryNormal: if err := g.applyEntry(e); err != nil { return err } case raftpb.EntryConfChange: if err := g.applyConfChange(e); err != nil { return err } default: glog.Fatalf("unexpected entry type") } g.applied = e.Index } if g.applied-g.snapped > g.snapCount { glog.Infof("%v start to snapshot (applied: %d, lastsnap: %d)", g, g.applied, g.snapped) g.snapshot() } return nil }
func (g *group) save(rdsv readySaved) error { glog.V(3).Infof("%v saving state", g) if rdsv.ready.SoftState != nil && rdsv.ready.SoftState.Lead != 0 { g.node.notifyElection(g.id) } // Apply snapshot to storage if it is more updated than current snapped. if !etcdraft.IsEmptySnap(rdsv.ready.Snapshot) { if err := g.diskStorage.SaveSnap(rdsv.ready.Snapshot); err != nil { glog.Fatalf("err in save snapshot: %v", err) } g.raftStorage.ApplySnapshot(rdsv.ready.Snapshot) glog.Infof("%v saved incoming snapshot at index %d", g, rdsv.ready.Snapshot.Metadata.Index) } err := g.diskStorage.Save(rdsv.ready.HardState, rdsv.ready.Entries) if err != nil { glog.Fatalf("err in raft storage save: %v", err) } glog.V(3).Infof("%v saved state on disk", g) g.raftStorage.Append(rdsv.ready.Entries) glog.V(3).Infof("%v appended entries in storage", g) // Apply config changes in the node as soon as possible // before applying other entries in the state machine. for _, e := range rdsv.ready.CommittedEntries { if e.Type != raftpb.EntryConfChange { continue } if e.Index <= g.saved { continue } g.saved = e.Index var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) if glog.V(2) { glog.Infof("%v applies conf change %s: %s", g, formatConfChange(cc), formatEntry(e)) } if err := g.validConfChange(cc); err != nil { glog.Errorf("%v received an invalid conf change for node %v: %v", g, cc.NodeID, err) cc.NodeID = etcdraft.None g.node.node.ApplyConfChange(g.id, cc) continue } cch := make(chan struct{}) go func() { g.confState = *g.node.node.ApplyConfChange(g.id, cc) close(cch) }() select { case <-g.node.done: return ErrStopped case <-cch: } } glog.V(3).Infof("%v successfully saved ready", g) rdsv.saved <- struct{}{} select { case g.applyc <- rdsv.ready: case <-g.node.done: } return nil }