func (s *Server) waitOnline() {
for {
info, err := s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.PanicErrorf(err, "get proxy info failed")
}
switch info.State {
case models.PROXY_STATE_MARK_OFFLINE:
s.handleMarkOffline()
case models.PROXY_STATE_ONLINE:
s.info.State = info.State
log.Infof("we are online: %s", s.info.Id)
_, err := s.topo.WatchNode(path.Join(models.GetProxyPath(s.topo.ProductName), s.info.Id), s.evtbus)
if err != nil {
log.PanicErrorf(err, "watch node failed")
}
return
}
select {
case e := <-s.evtbus:
switch e.(type) {
case *killEvent:
s.handleMarkOffline()
}
default: //otherwise ignore it
}
log.Warnf("wait to be online: %s", s.info.Id)
time.Sleep(3 * time.Second)
}
}
func (s *Server) processAction(e interface{}) {
if strings.Index(getEventPath(e), models.GetProxyPath(s.topo.ProductName)) == 0 {
info, err := s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.PanicErrorf(err, "get proxy info failed: %s", s.info.Id)
}
switch info.State {
case models.PROXY_STATE_MARK_OFFLINE:
log.Infof("mark offline, proxy got offline event: %s", s.info.Id)
s.markOffline()
case models.PROXY_STATE_ONLINE:
s.rewatchProxy()
default:
log.Panicf("unknown proxy state %v", info)
}
return
}
//re-watch
nodes := s.rewatchNodes()
seqs, err := models.ExtraSeqList(nodes)
if err != nil {
log.PanicErrorf(err, "get seq list failed")
}
if len(seqs) == 0 || !s.topo.IsChildrenChangedEvent(e) {
return
}
//get last pos
index := -1
for i, seq := range seqs {
if s.lastActionSeq < seq {
index = i
//break
//only handle latest action
}
}
if index < 0 {
return
}
actions := seqs[index:]
for _, seq := range actions {
exist, err := s.topo.Exist(path.Join(s.topo.GetActionResponsePath(seq), s.info.Id))
if err != nil {
log.PanicErrorf(err, "get action failed")
}
if exist {
continue
}
if s.checkAndDoTopoChange(seq) {
s.responseAction(int64(seq))
}
}
s.lastActionSeq = seqs[len(seqs)-1]
}
func (s *Server) waitOnline() {
s.mu.Lock()
defer s.mu.Unlock()
for {
pi, err := s.top.GetProxyInfo(s.pi.Id)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
if pi.State == models.PROXY_STATE_MARK_OFFLINE {
s.handleMarkOffline()
}
if pi.State == models.PROXY_STATE_ONLINE {
s.pi.State = pi.State
println("good, we are on line", s.pi.Id)
log.Info("we are online", s.pi.Id)
_, err := s.top.WatchNode(path.Join(models.GetProxyPath(s.top.ProductName), s.pi.Id), s.evtbus)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
return
}
println("wait to be online ", s.pi.Id)
log.Warning(s.pi.Id, "wait to be online")
time.Sleep(3 * time.Second)
}
}
func (s *Server) processAction(e interface{}) {
if strings.Index(getEventPath(e), models.GetProxyPath(s.topo.ProductName)) == 0 {
info, err := s.topo.GetProxyInfo(s.info.Id)
if err != nil {
log.PanicErrorf(err, "get proxy info failed: %s", s.info.Id)
}
if info.State == models.PROXY_STATE_MARK_OFFLINE {
s.handleMarkOffline()
}
return
}
//re-watch
nodes, err := s.topo.WatchChildren(models.GetWatchActionPath(s.topo.ProductName), s.evtbus)
if err != nil {
log.PanicErrorf(err, "rewatch children failed")
}
seqs, err := models.ExtraSeqList(nodes)
if err != nil {
log.PanicErrorf(err, "get seq list failed")
}
if len(seqs) == 0 || !s.topo.IsChildrenChangedEvent(e) {
return
}
//get last pos
index := -1
for i, seq := range seqs {
if s.lastActionSeq < seq {
index = i
break
}
}
if index < 0 {
return
}
actions := seqs[index:]
for _, seq := range actions {
exist, err := s.topo.Exist(path.Join(s.topo.GetActionResponsePath(seq), s.info.Id))
if err != nil {
log.PanicErrorf(err, "get action failed")
}
if exist {
continue
}
if s.checkAndDoTopoChange(seq) {
s.responseAction(int64(seq))
}
}
s.lastActionSeq = seqs[len(seqs)-1]
}
func (top *Topology) Close(proxyName string) {
// delete fence znode
pi, err := models.GetProxyInfo(top.zkConn, top.ProductName, proxyName)
if err != nil {
log.Errorf("killing fence error, proxy %s is not exists", proxyName)
} else {
zkhelper.DeleteRecursive(top.zkConn, path.Join(models.GetProxyFencePath(top.ProductName), pi.Addr), -1)
}
// delete ephemeral znode
zkhelper.DeleteRecursive(top.zkConn, path.Join(models.GetProxyPath(top.ProductName), proxyName), -1)
top.zkConn.Close()
}
func (s *Server) processAction(e interface{}) {
if strings.Index(GetEventPath(e), models.GetProxyPath(s.top.ProductName)) == 0 {
//proxy event, should be order for me to suicide
s.handleProxyCommand()
return
}
//re-watch
nodes, err := s.top.WatchChildren(models.GetWatchActionPath(s.top.ProductName), s.evtbus)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
seqs, err := models.ExtraSeqList(nodes)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
if len(seqs) == 0 || !s.top.IsChildrenChangedEvent(e) {
return
}
//get last pos
index := -1
for i, seq := range seqs {
if s.lastActionSeq < seq {
index = i
break
}
}
if index < 0 {
return
}
actions := seqs[index:]
for _, seq := range actions {
exist, err := s.top.Exist(path.Join(s.top.GetActionResponsePath(seq), s.pi.Id))
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
if exist {
continue
}
if s.checkAndDoTopoChange(seq) {
s.responseAction(int64(seq))
}
}
s.lastActionSeq = seqs[len(seqs)-1]
}
func (s *Server) rewatchProxy(invokeFromRestart bool) {
var err error
for {
_, err = s.topo.WatchNode(path.Join(models.GetProxyPath(s.topo.ProductName), s.info.Id), s.evtbus)
if err != nil {
log.ErrorErrorf(err, "watch node failed")
if s.topo.IsFatalErr(err) {
s.reRegister(models.PROXY_STATE_ONLINE, invokeFromRestart)
} else {
time.Sleep(5 * time.Second)
}
} else {
break
}
}
}
func (s *Server) waitOnline() {
for {
pi, err := s.top.GetProxyInfo(s.pi.Id)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
if pi.State == models.PROXY_STATE_MARK_OFFLINE {
s.handleMarkOffline()
}
if pi.State == models.PROXY_STATE_ONLINE {
s.pi.State = pi.State
println("good, we are on line", s.pi.Id)
log.Info("we are online", s.pi.Id)
_, err := s.top.WatchNode(path.Join(models.GetProxyPath(s.top.ProductName), s.pi.Id), s.evtbus)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
return
}
select {
case e := <-s.evtbus:
switch e.(type) {
case *killEvent:
s.handleMarkOffline()
e.(*killEvent).done <- nil
}
default: //otherwise ignore it
}
println("wait to be online ", s.pi.Id)
log.Warning(s.pi.Id, "wait to be online")
time.Sleep(3 * time.Second)
}
}
func (s *Server) rewatchProxy() {
_, err := s.topo.WatchNode(path.Join(models.GetProxyPath(s.topo.ProductName), s.info.Id), s.evtbus)
if err != nil {
log.PanicErrorf(err, "watch node failed")
}
}
func (top *Topology) Close(proxyName string) {
zkhelper.DeleteRecursive(top.zkConn, path.Join(models.GetProxyPath(top.ProductName), proxyName), -1)
top.zkConn.Close()
}
func (s *Server) processAction(e interface{}) {
start := time.Now()
s.mu.Lock()
defer s.mu.Unlock()
if time.Since(start).Seconds() > 10 {
log.Warning("take too long to get lock")
}
actPath := GetEventPath(e)
if strings.Index(actPath, models.GetProxyPath(s.top.ProductName)) == 0 {
//proxy event, should be order for me to suicide
s.handleProxyCommand()
return
}
//re-watch
nodes, err := s.top.WatchChildren(models.GetWatchActionPath(s.top.ProductName), s.evtbus)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
seqs, err := models.ExtraSeqList(nodes)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
if len(seqs) == 0 || !s.top.IsChildrenChangedEvent(e) {
return
}
//get last pos
index := -1
for i, seq := range seqs {
if s.lastActionSeq < seq {
index = i
break
}
}
if index < 0 {
log.Warningf("zookeeper restarted or actions were deleted ? lastActionSeq: %d", s.lastActionSeq)
if s.lastActionSeq > seqs[len(seqs)-1] {
log.Fatalf("unknown error, zookeeper restarted or actions were deleted ? lastActionSeq: %d, %v", s.lastActionSeq, nodes)
}
if s.lastActionSeq == seqs[len(seqs)-1] { //children change or delete event
return
}
//actions node was remove by someone, seems we can handle it
index = 0
}
actions := seqs[index:]
for _, seq := range actions {
exist, err := s.top.Exist(path.Join(s.top.GetActionResponsePath(seq), s.pi.Id))
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
if exist {
continue
}
if s.checkAndDoTopoChange(seq) {
s.responseAction(int64(seq))
}
}
s.lastActionSeq = seqs[len(seqs)-1]
}