func (s *Server) checkAndDoTopoChange(seq int) bool {
act, err := s.topo.GetActionWithSeq(int64(seq))
if err != nil { //todo: error is not "not exist"
log.PanicErrorf(err, "action failed, seq = %d", seq)
}
if !needResponse(act.Receivers, s.info) { //no need to response
return false
}
log.Warnf("action %v receivers %v", seq, act.Receivers)
switch act.Type {
case models.ACTION_TYPE_SLOT_MIGRATE, models.ACTION_TYPE_SLOT_CHANGED,
models.ACTION_TYPE_SLOT_PREMIGRATE:
slot := &models.Slot{}
s.getActionObject(seq, slot)
s.fillSlot(slot.Id)
case models.ACTION_TYPE_SERVER_GROUP_CHANGED:
serverGroup := &models.ServerGroup{}
s.getActionObject(seq, serverGroup)
s.onGroupChange(serverGroup.Id)
case models.ACTION_TYPE_SERVER_GROUP_REMOVE:
//do not care
case models.ACTION_TYPE_MULTI_SLOT_CHANGED:
param := &models.SlotMultiSetParam{}
s.getActionObject(seq, param)
s.onSlotRangeChange(param)
default:
log.Panicf("unknown action %+v", act)
}
return true
}
func (top *Topology) doWatch(evtch <-chan topo.Event, evtbus chan interface{}) {
e := <-evtch
if e.State == topo.StateExpired || e.Type == topo.EventNotWatching {
log.Panicf("session expired: %+v", e)
}
log.Warnf("topo event %+v", e)
switch e.Type {
//case topo.EventNodeCreated:
//case topo.EventNodeDataChanged:
case topo.EventNodeChildrenChanged: //only care children changed
//todo:get changed node and decode event
default:
log.Warnf("%+v", e)
}
evtbus <- e
}
func (s *Slot) prepare(r *Request, key []byte) (*SharedBackendConn, error) {
if s.backend.bc == nil {
log.Infof("slot-%04d is not ready: key = %s", s.id, key)
return nil, ErrSlotIsNotReady
}
if err := s.slotsmgrt(r, key); err != nil {
log.Warnf("slot-%04d migrate from = %s to %s failed: key = %s, error = %s",
s.id, s.migrate.from, s.backend.addr, key, err)
return nil, err
} else {
r.slot = &s.wait
r.slot.Add(1)
return s.backend.bc, nil
}
}
func WaitForReceiverWithTimeout(zkConn zkhelper.Conn, productName string, actionZkPath string, proxies []ProxyInfo, timeoutInMs int) error {
if len(proxies) == 0 {
return nil
}
times := 0
proxyIds := make(map[string]bool)
for _, p := range proxies {
proxyIds[p.Id] = true
}
// check every 500ms
for times < timeoutInMs/500 {
if times >= 6 && (times*500)%1000 == 0 {
log.Warnf("abnormal waiting time for receivers: %s %v", actionZkPath, proxyIds)
}
// get confirm ids
nodes, _, err := zkConn.Children(actionZkPath)
if err != nil {
return errors.Trace(err)
}
for _, node := range nodes {
id := path.Base(node)
delete(proxyIds, id)
}
if len(proxyIds) == 0 {
return nil
}
times++
time.Sleep(500 * time.Millisecond)
}
log.Warn("proxies didn't responed: ", proxyIds)
// set offline proxies
for id, _ := range proxyIds {
log.Errorf("mark proxy %s to PROXY_STATE_MARK_OFFLINE", id)
if err := SetProxyStatus(zkConn, productName, id, PROXY_STATE_MARK_OFFLINE); err != nil {
return errors.Trace(err)
}
}
return ErrReceiverTimeout
}
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error {
// set slot status
s, err := models.GetSlot(t.zkConn, t.productName, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info failed")
return err
}
if s.State.Status == models.SLOT_STATUS_OFFLINE {
log.Warnf("status is offline: %+v", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
// make sure from group & target group exists
exists, err := models.GroupExists(t.zkConn, t.productName, from)
if err != nil {
return errors.Trace(err)
}
if !exists {
log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to)
return errors.Errorf("group %d not found", from)
}
exists, err = models.GroupExists(t.zkConn, t.productName, to)
if err != nil {
return errors.Trace(err)
}
if !exists {
return errors.Errorf("group %d not found", to)
}
// cannot migrate to itself, just ignore
if from == to {
log.Warnf("from == to, ignore: %+v", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil {
log.ErrorErrorf(err, "set migrate status failed")
return err
}
err = t.Migrate(s, from, to, func(p SlotMigrateProgress) {
// on migrate slot progress
if p.Remain%5000 == 0 {
log.Infof("%+v", p)
}
})
if err != nil {
log.ErrorErrorf(err, "migrate slot failed")
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(t.zkConn); err != nil {
log.ErrorErrorf(err, "update zk status failed, should be: %+v", s)
return err
}
return nil
}