func (md *MasterDetector) detect(f detector.MasterChanged) {
log.V(3).Infoln("detecting children at", CurrentPath)
detectLoop:
for {
select {
case <-md.Done():
return
default:
}
log.V(3).Infoln("watching children at", CurrentPath)
path, childrenCh, errCh := md.client.WatchChildren(CurrentPath)
rewatch := false
for {
started := time.Now()
select {
case children := <-childrenCh:
md.childrenChanged(path, children, f)
case err, ok := <-errCh:
// check for a tie first (required for predictability (tests)); the downside of
// doing this is that a listener might get two callbacks back-to-back ("new leader",
// followed by "no leader").
select {
case children := <-childrenCh:
md.childrenChanged(path, children, f)
default:
}
if ok {
log.V(1).Infoln("child watch ended with error, master lost; error was:", err.Error())
} else {
// detector shutdown likely...
log.V(1).Infoln("child watch ended, master lost")
}
select {
case <-md.Done():
return
default:
if md.leaderNode != "" {
log.V(2).Infof("changing leader node from %q -> \"\"", md.leaderNode)
md.leaderNode = ""
f.OnMasterChanged(nil)
}
}
rewatch = true
}
// rate-limit master changes
if elapsed := time.Now().Sub(started); elapsed > 0 {
log.V(2).Infoln("resting before next detection cycle")
select {
case <-md.Done():
return
case <-time.After(md.minDetectorCyclePeriod - elapsed): // noop
}
}
if rewatch {
continue detectLoop
}
}
}
}
func (m *MockDetector) Detect(listener detector.MasterChanged) error {
if listener != nil {
if pid, err := upid.Parse("master(2)@" + m.address); err != nil {
return err
} else {
go listener.OnMasterChanged(detector.CreateMasterInfo(pid))
}
}
return nil
}
func (md *MasterDetector) detect(f detector.MasterChanged) {
minCyclePeriod := 1 * time.Second
detectLoop:
for {
started := time.Now()
select {
case <-md.Done():
return
case <-md.client.connections():
// we let the golang runtime manage our listener list for us, in form of goroutines that
// callback to the master change notification listen func's
if watchEnded, err := md.client.watchChildren(currentPath, ChildWatcher(func(zkc *Client, path string) {
md.childrenChanged(zkc, path, f)
})); err == nil {
log.V(2).Infoln("detector listener installed")
select {
case <-watchEnded:
if md.leaderNode != "" {
log.V(1).Infof("child watch ended, signaling master lost")
md.leaderNode = ""
f.OnMasterChanged(nil)
}
case <-md.client.stopped():
return
}
} else {
log.V(1).Infof("child watch ended with error: %v", err)
continue detectLoop
}
}
// rate-limit master changes
if elapsed := time.Now().Sub(started); elapsed > 0 {
log.V(2).Infoln("resting before next detection cycle")
select {
case <-md.Done():
return
case <-time.After(minCyclePeriod - elapsed): // noop
}
}
}
}
func (md *MasterDetector) notifyMasterChanged(path string, list []string, obs detector.MasterChanged) {
topNode := selectTopNode(list)
if md.leaderNode == topNode {
log.V(2).Infof("ignoring children-changed event, leader has not changed: %v", path)
return
}
log.V(2).Infof("changing leader node from %s -> %s", md.leaderNode, topNode)
md.leaderNode = topNode
var masterInfo *mesos.MasterInfo
if md.leaderNode != "" {
var err error
if masterInfo, err = md.pullMasterInfo(path, topNode); err != nil {
log.Errorln(err.Error())
}
}
log.V(2).Infof("detected master info: %+v", masterInfo)
logPanic(func() { obs.OnMasterChanged(masterInfo) })
}
func (md *MasterDetector) notifyMasterChanged(path string, list []string, obs detector.MasterChanged) {
// mesos v0.24 writes JSON only, v0.23 writes json and protobuf, v0.22 and prior only write protobuf
topNode, codec := md.selectTopNode(list)
if md.leaderNode == topNode {
log.V(2).Infof("ignoring children-changed event, leader has not changed: %v", path)
return
}
log.V(2).Infof("changing leader node from %q -> %q", md.leaderNode, topNode)
md.leaderNode = topNode
var masterInfo *mesos.MasterInfo
if md.leaderNode != "" {
var err error
if masterInfo, err = codec(path, topNode); err != nil {
log.Errorln(err.Error())
}
}
log.V(2).Infof("detected master info: %+v", masterInfo)
logPanic(func() { obs.OnMasterChanged(masterInfo) })
}
//TODO(jdef) execute async because we don't want to stall our client's event loop? if so
//then we also probably want serial event delivery (aka. delivery via a chan) but then we
//have to deal with chan buffer sizes .. ugh. This is probably the least painful for now.
func (md *MasterDetector) childrenChanged(zkc *Client, path string, obs detector.MasterChanged) {
log.V(2).Infof("fetching children at path '%v'", path)
list, err := zkc.list(path)
if err != nil {
log.Warning(err)
return
}
topNode := selectTopNode(list)
if md.leaderNode == topNode {
log.V(2).Infof("ignoring children-changed event, leader has not changed: %v", path)
return
}
log.V(2).Infof("changing leader node from %s -> %s", md.leaderNode, topNode)
md.leaderNode = topNode
var masterInfo *mesos.MasterInfo
if md.leaderNode != "" {
data, err := zkc.data(fmt.Sprintf("%s/%s", path, topNode))
if err != nil {
log.Errorf("unable to retrieve leader data: %v", err.Error())
return
}
masterInfo = new(mesos.MasterInfo)
err = proto.Unmarshal(data, masterInfo)
if err != nil {
log.Errorf("unable to unmarshall MasterInfo data from zookeeper: %v", err)
return
}
}
log.V(2).Infof("detected master info: %+v", masterInfo)
obs.OnMasterChanged(masterInfo)
}
func (md FakeMasterDetector) Detect(cb detector.MasterChanged) error {
md.callback = cb
leadingMaster := mesosutil.NewMasterInfo(TEST_MASTER_ID, TEST_MASTER_IP, TEST_MASTER_PORT)
cb.OnMasterChanged(leadingMaster)
return nil
}
