// MembershipUpdateAfter generates the Flatbuffer response containing all the
// membership updates after the provided raft index.
func (g *groupi) MembershipUpdateAfter(ridx uint64) *task.MembershipUpdate {
g.RLock()
defer g.RUnlock()
maxIdx := ridx
out := new(task.MembershipUpdate)
for gid, peers := range g.all {
for _, s := range peers.list {
if s.RaftIdx <= ridx {
continue
}
if s.RaftIdx > maxIdx {
maxIdx = s.RaftIdx
}
out.Members = append(out.Members,
&task.Membership{
Leader: s.Leader,
Id: s.NodeId,
GroupId: gid,
Addr: s.Addr,
})
}
}
out.LastUpdate = maxIdx
return out
}
// syncMemberships needs to be called in an periodic loop.
// How syncMemberships works:
// - Each server iterates over all the nodes it's serving, present in local.
// - If serving group zero, propose membership status updates directly via RAFT.
// - Otherwise, generates a membership update, which includes status of all serving nodes.
// - Check if it has address of a server from group zero. If so, use that.
// - Otherwise, use the peer information passed down via flags.
// - Send update via UpdateMembership call to the peer.
// - If the peer doesn't serve group zero, it would return back a redirect with the right address.
// - Otherwise, it would iterate over the memberships, check for duplicates, and apply updates.
// - Once iteration is over without errors, it would return back all new updates.
// - These updates are then applied to groups().all state via applyMembershipUpdate.
func (g *groupi) syncMemberships() {
if g.ServesGroup(0) {
// This server serves group zero.
g.RLock()
defer g.RUnlock()
for _, n := range g.local {
rc := n.raftContext
if g.duplicate(rc.Group, rc.Id, rc.Addr, n.AmLeader()) {
continue
}
go func(rc *task.RaftContext, amleader bool) {
mm := &task.Membership{
Leader: amleader,
Id: rc.Id,
GroupId: rc.Group,
Addr: rc.Addr,
}
zero := g.Node(0)
x.AssertTruef(zero != nil, "Expected node 0")
if err := zero.ProposeAndWait(zero.ctx, &task.Proposal{Membership: mm}); err != nil {
x.TraceError(g.ctx, err)
}
}(rc, n.AmLeader())
}
return
}
// This server doesn't serve group zero.
// Generate membership update of all local nodes.
var mu task.MembershipUpdate
{
g.RLock()
for _, n := range g.local {
rc := n.raftContext
mu.Members = append(mu.Members,
&task.Membership{
Leader: n.AmLeader(),
Id: rc.Id,
GroupId: rc.Group,
Addr: rc.Addr,
})
}
mu.LastUpdate = g.lastUpdate
g.RUnlock()
}
// Send an update to peer.
var pl *pool
addr := g.AnyServer(0)
UPDATEMEMBERSHIP:
if len(addr) > 0 {
pl = pools().get(addr)
} else {
pl = pools().any()
}
conn, err := pl.Get()
if err == errNoConnection {
fmt.Println("Unable to sync memberships. No valid connection")
return
}
x.Check(err)
defer pl.Put(conn)
c := NewWorkerClient(conn)
update, err := c.UpdateMembership(g.ctx, &mu)
if err != nil {
x.TraceError(g.ctx, err)
return
}
// Check if we got a redirect.
if update.Redirect {
addr = update.RedirectAddr
if len(addr) == 0 {
return
}
fmt.Printf("Got redirect for: %q\n", addr)
pools().connect(addr)
goto UPDATEMEMBERSHIP
}
var lu uint64
for _, mm := range update.Members {
g.applyMembershipUpdate(update.LastUpdate, mm)
if lu < update.LastUpdate {
lu = update.LastUpdate
}
}
g.TouchLastUpdate(lu)
}