func isLeader(l lease.Lease, machID string) bool {
if l == nil {
return false
}
if l.MachineID() != machID {
return false
}
return true
}
func renewLeadership(l lease.Lease, ttl time.Duration) lease.Lease {
err := l.Renew(ttl)
if err != nil {
log.Errorf("sentinel leadership lost, renewal failed: %v", err)
return nil
}
log.Debugf("sentinel leadership renewed")
return l
}
func (s *Sentinel) clusterSentinelCheck(pctx context.Context) {
s.updateMutex.Lock()
defer s.updateMutex.Unlock()
e := s.e
cd, res, err := e.GetClusterData()
if err != nil {
log.Errorf("error retrieving cluster data: %v", err)
return
}
var prevCDIndex uint64
if res != nil {
prevCDIndex = res.Node.ModifiedIndex
}
var cv *cluster.ClusterView
var keepersState cluster.KeepersState
if cd == nil {
cv = cluster.NewClusterView()
keepersState = nil
} else {
cv = cd.ClusterView
keepersState = cd.KeepersState
}
log.Debugf(spew.Sprintf("keepersState: %#v", keepersState))
log.Debugf(spew.Sprintf("clusterView: %#v", cv))
// Update cluster config
// This shouldn't need a lock
s.clusterConfig = cv.Config.ToConfig()
if err := s.setSentinelInfo(2 * s.clusterConfig.SleepInterval); err != nil {
log.Errorf("cannot update leader sentinel info: %v", err)
return
}
// TODO(sgotti) better ways to calculate leaseTTL?
leaseTTL := s.clusterConfig.SleepInterval + s.clusterConfig.RequestTimeout*4
ctx, cancel := context.WithTimeout(pctx, s.clusterConfig.RequestTimeout)
keepersDiscoveryInfo, err := s.discover(ctx)
cancel()
if err != nil {
log.Errorf("err: %v", err)
return
}
log.Debugf(spew.Sprintf("keepersDiscoveryInfo: %#v", keepersDiscoveryInfo))
ctx, cancel = context.WithTimeout(pctx, s.clusterConfig.RequestTimeout)
keepersInfo, err := getKeepersInfo(ctx, keepersDiscoveryInfo)
cancel()
if err != nil {
log.Errorf("err: %v", err)
return
}
log.Debugf(spew.Sprintf("keepersInfo: %#v", keepersInfo))
ctx, cancel = context.WithTimeout(pctx, s.clusterConfig.RequestTimeout)
keepersPGState := getKeepersPGState(ctx, keepersInfo)
cancel()
log.Debugf(spew.Sprintf("keepersPGState: %#v", keepersPGState))
var l lease.Lease
if isLeader(s.l, s.id) {
log.Infof("I'm the sentinels leader")
l = renewLeadership(s.l, leaseTTL)
} else {
log.Infof("trying to acquire sentinels leadership")
l = acquireLeadership(s.lManager, s.id, 1, leaseTTL)
}
// log all leadership changes
if l != nil && s.l == nil && l.MachineID() != s.id {
log.Infof("sentinel leader is %s", l.MachineID())
} else if l != nil && s.l != nil && l.MachineID() != l.MachineID() {
log.Infof("sentinel leadership changed from %s to %s", l.MachineID(), l.MachineID())
}
s.l = l
if !isLeader(s.l, s.id) {
return
}
if err := s.setLeaderSentinelInfo(leaseTTL); err != nil {
log.Errorf("cannot update leader sentinel info: %v", err)
return
}
if cv.Version == 0 {
// Cluster first initialization
newcv := cluster.NewClusterView()
newcv.Version = 1
_, err = e.SetClusterData(nil, newcv, 0)
if err != nil {
log.Errorf("error saving clusterdata: %v", err)
}
return
}
newKeepersState := s.updateKeepersState(keepersState, keepersInfo, keepersPGState)
log.Debugf(spew.Sprintf("newKeepersState: %#v", newKeepersState))
newcv, err := s.updateClusterView(cv, newKeepersState)
if err != nil {
log.Errorf("failed to update clusterView: %v", err)
return
}
log.Debugf(spew.Sprintf("newcv: %#v", newcv))
if cv.Version < newcv.Version {
log.Debugf("newcv changed from previous cv")
}
_, err = e.SetClusterData(newKeepersState, newcv, prevCDIndex)
if err != nil {
log.Errorf("error saving clusterdata: %v", err)
}
}
func (s *Sentinel) clusterSentinelSM(pctx context.Context) {
e := s.e
// Update cluster config
clusterConfig, _, err := e.GetClusterConfig()
if err != nil {
log.Errorf("cannot get cluster config: %v", err)
return
}
log.Debugf(spew.Sprintf("clusterConfig: %#v", clusterConfig))
// This shouldn't need a lock
s.clusterConfig = clusterConfig
// TODO(sgotti) better ways to calculate leaseTTL?
leaseTTL := clusterConfig.SleepInterval + clusterConfig.RequestTimeout*4
ctx, cancel := context.WithTimeout(pctx, s.clusterConfig.RequestTimeout)
membersDiscoveryInfo, err := s.discover(ctx)
cancel()
if err != nil {
log.Errorf("err: %v", err)
return
}
log.Debugf(spew.Sprintf("membersDiscoveryInfo: %#v", membersDiscoveryInfo))
ctx, cancel = context.WithTimeout(pctx, s.clusterConfig.RequestTimeout)
membersInfo, err := getMembersInfo(ctx, membersDiscoveryInfo)
cancel()
if err != nil {
log.Errorf("err: %v", err)
return
}
log.Debugf(spew.Sprintf("membersInfo: %#v", membersInfo))
ctx, cancel = context.WithTimeout(pctx, s.clusterConfig.RequestTimeout)
membersPGState := getMembersPGState(ctx, membersInfo)
cancel()
log.Debugf(spew.Sprintf("membersPGState: %#v", membersPGState))
var l lease.Lease
if isLeader(s.l, s.id) {
log.Infof("I'm the sentinels leader")
l = renewLeadership(s.l, leaseTTL)
} else {
log.Infof("trying to acquire sentinels leadership")
l = acquireLeadership(s.lManager, s.id, 1, leaseTTL)
}
// log all leadership changes
if l != nil && s.l == nil && l.MachineID() != s.id {
log.Infof("sentinel leader is %s", l.MachineID())
} else if l != nil && s.l != nil && l.MachineID() != l.MachineID() {
log.Infof("sentinel leadership changed from %s to %s", l.MachineID(), l.MachineID())
}
s.l = l
if !isLeader(s.l, s.id) {
return
}
cd, res, err := e.GetClusterData()
if err != nil {
log.Errorf("error retrieving cluster data: %v", err)
return
}
var prevCDIndex uint64
if res != nil {
prevCDIndex = res.Node.ModifiedIndex
}
var cv *cluster.ClusterView
var membersState cluster.MembersState
if cd == nil {
cv = cluster.NewClusterView()
membersState = nil
} else {
cv = cd.ClusterView
membersState = cd.MembersState
}
log.Debugf(spew.Sprintf("membersState: %#v", membersState))
log.Debugf(spew.Sprintf("clusterView: %#v", cv))
pv, res, err := e.GetProxyView()
if err != nil {
log.Errorf("err: %v", err)
return
}
log.Debugf(spew.Sprintf("proxyview: %#v", pv))
var prevPVIndex uint64
if res != nil {
prevPVIndex = res.Node.ModifiedIndex
}
newMembersState := s.updateMembersState(membersState, membersInfo, membersPGState)
log.Debugf(spew.Sprintf("newMembersState: %#v", newMembersState))
newcv, err := s.updateClusterView(cv, newMembersState)
if err != nil {
log.Errorf("failed to update clusterView: %v", err)
return
}
log.Debugf(spew.Sprintf("newcv: %#v", newcv))
if cv.Version < newcv.Version {
log.Debugf("newcv changed from previous cv")
if err := s.updateProxyView(cv, newcv, newMembersState, prevPVIndex); err != nil {
log.Errorf("error updating proxyView: %v", err)
return
}
}
_, err = e.SetClusterData(newMembersState, newcv, prevCDIndex)
if err != nil {
log.Errorf("error saving clusterdata: %v", err)
}
}