func report(err error, batch *raft.Batch, r raft.Reporter) {
for g, msgs := range batch.Messages {
if unreachable(err) {
r.ReportUnreachable(batch.To, g)
}
for _, msg := range msgs {
if !etcdraft.IsEmptySnap(msg.Snapshot) {
r.ReportSnapshot(batch.To, g, snapStatus(err))
}
}
}
}
func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error {
if raft.IsEmptySnap(snapshot) {
return nil
}
return s.save(&snapshot)
}
func (n *MultiNode) handleReadies(readies map[uint64]etcdraft.Ready) {
glog.V(3).Infof("%v handles a ready", n)
beatBatch := make(nodeBatch)
normBatch := make(nodeBatch)
snapBatch := make(nodeBatch)
saved := make(chan struct{}, len(readies))
for gid, rd := range readies {
if !shouldSave(rd) {
saved <- struct{}{}
continue
}
g, ok := n.groups[gid]
if !ok {
glog.Fatalf("cannot find group %v", g)
}
g.savec <- readySaved{
ready: rd,
saved: saved,
}
}
for gid, rd := range readies {
for _, m := range rd.Messages {
if m.Type == raftpb.MsgHeartbeat || m.Type == raftpb.MsgHeartbeatResp {
batch := beatBatch.batch(m.To)
// Only one heartbeat/response message should suffice.
batch.Messages[gid] = []raftpb.Message{m}
continue
}
var batch *Batch
if !etcdraft.IsEmptySnap(m.Snapshot) {
batch = snapBatch.batch(m.To)
} else {
batch = normBatch.batch(m.To)
}
batch.Messages[gid] = append(batch.Messages[gid], m)
}
}
for i := 0; i < len(readies); i++ {
select {
case <-saved:
case <-n.done:
return
}
}
glog.V(3).Infof("%v saved the readies for all groups", n)
for nid, batch := range beatBatch {
glog.V(3).Infof("%v sends high priority batch to %v", n, nid)
batch.From = n.id
batch.To = nid
batch.Priority = High
n.send(batch, n.node)
}
for nid, batch := range normBatch {
glog.V(3).Infof("%v sends normal priority batch to %v", n, nid)
batch.From = n.id
batch.To = nid
batch.Priority = Normal
n.send(batch, n.node)
}
for nid, batch := range snapBatch {
glog.V(3).Infof("%v sends low priority batch to %v", n, nid)
batch.From = n.id
batch.To = nid
batch.Priority = Low
n.send(batch, n.node)
}
select {
case n.advancec <- readies:
case <-n.done:
}
}
func shouldSave(rd etcdraft.Ready) bool {
return rd.SoftState != nil || !etcdraft.IsEmptyHardState(rd.HardState) ||
!etcdraft.IsEmptySnap(rd.Snapshot) || len(rd.Entries) > 0 ||
len(rd.CommittedEntries) > 0
}
func (g *group) apply(ready etcdraft.Ready) error {
if ready.SoftState != nil {
newLead := ready.SoftState.Lead
if g.leader != newLead {
g.stateMachine.ProcessStatusChange(LeaderChanged{
Old: g.leader,
New: newLead,
Term: ready.HardState.Term,
})
g.leader = newLead
}
}
// Recover from snapshot if it is more recent than the currently
// applied.
if !etcdraft.IsEmptySnap(ready.Snapshot) &&
ready.Snapshot.Metadata.Index > g.applied {
if err := g.stateMachine.Restore(ready.Snapshot.Data); err != nil {
glog.Fatalf("error in recovering the state machine: %v", err)
}
// FIXME(soheil): update the nodes and notify the application?
g.applied = ready.Snapshot.Metadata.Index
glog.Infof("%v recovered from incoming snapshot at index %d", g.node,
g.snapped)
}
es := ready.CommittedEntries
if len(es) == 0 {
return nil
}
firsti := es[0].Index
if firsti > g.applied+1 {
glog.Fatalf(
"1st index of committed entry[%d] should <= applied[%d] + 1",
firsti, g.applied)
}
if glog.V(3) {
glog.Infof("%v receives raft update: committed=%s appended=%s", g,
formatEntries(es), formatEntries(ready.Entries))
}
for _, e := range es {
if e.Index <= g.applied {
continue
}
switch e.Type {
case raftpb.EntryNormal:
if err := g.applyEntry(e); err != nil {
return err
}
case raftpb.EntryConfChange:
if err := g.applyConfChange(e); err != nil {
return err
}
default:
glog.Fatalf("unexpected entry type")
}
g.applied = e.Index
}
if g.applied-g.snapped > g.snapCount {
glog.Infof("%v start to snapshot (applied: %d, lastsnap: %d)", g,
g.applied, g.snapped)
g.snapshot()
}
return nil
}
func (g *group) save(rdsv readySaved) error {
glog.V(3).Infof("%v saving state", g)
if rdsv.ready.SoftState != nil && rdsv.ready.SoftState.Lead != 0 {
g.node.notifyElection(g.id)
}
// Apply snapshot to storage if it is more updated than current snapped.
if !etcdraft.IsEmptySnap(rdsv.ready.Snapshot) {
if err := g.diskStorage.SaveSnap(rdsv.ready.Snapshot); err != nil {
glog.Fatalf("err in save snapshot: %v", err)
}
g.raftStorage.ApplySnapshot(rdsv.ready.Snapshot)
glog.Infof("%v saved incoming snapshot at index %d", g,
rdsv.ready.Snapshot.Metadata.Index)
}
err := g.diskStorage.Save(rdsv.ready.HardState, rdsv.ready.Entries)
if err != nil {
glog.Fatalf("err in raft storage save: %v", err)
}
glog.V(3).Infof("%v saved state on disk", g)
g.raftStorage.Append(rdsv.ready.Entries)
glog.V(3).Infof("%v appended entries in storage", g)
// Apply config changes in the node as soon as possible
// before applying other entries in the state machine.
for _, e := range rdsv.ready.CommittedEntries {
if e.Type != raftpb.EntryConfChange {
continue
}
if e.Index <= g.saved {
continue
}
g.saved = e.Index
var cc raftpb.ConfChange
pbutil.MustUnmarshal(&cc, e.Data)
if glog.V(2) {
glog.Infof("%v applies conf change %s: %s",
g, formatConfChange(cc), formatEntry(e))
}
if err := g.validConfChange(cc); err != nil {
glog.Errorf("%v received an invalid conf change for node %v: %v",
g, cc.NodeID, err)
cc.NodeID = etcdraft.None
g.node.node.ApplyConfChange(g.id, cc)
continue
}
cch := make(chan struct{})
go func() {
g.confState = *g.node.node.ApplyConfChange(g.id, cc)
close(cch)
}()
select {
case <-g.node.done:
return ErrStopped
case <-cch:
}
}
glog.V(3).Infof("%v successfully saved ready", g)
rdsv.saved <- struct{}{}
select {
case g.applyc <- rdsv.ready:
case <-g.node.done:
}
return nil
}