/**
* When allocation changed, modify the state of slot and update slotmaps.
*/
func HandleAllocationChange(oldAllocations, newAllocations *SlotAllocation, slotinfoMaps *SlotInfoMaps, zkHelper *utils.ZkHelper) {
isChanged := false
newSlotInfoMap := make(map[string]*SlotInfo) //create a new slotinfo maps
for i := 0; i < oldAllocations.SlotCount; i++ {
oldNodeId := oldAllocations.Allocations[strconv.Itoa(i)]
newNodeId := newAllocations.Allocations[strconv.Itoa(i)]
newSlotInfoMap[strconv.Itoa(i)] = slotinfoMaps.GetSlotInfoMap()[strconv.Itoa(i)].Clone()
if oldNodeId != newNodeId {
isChanged = true
log.Infof("The slot %d's node changed to %d from %d.", i, newNodeId, oldNodeId)
newSlotInfoMap[strconv.Itoa(i)].MigrateState = MigStateMigrating
newSlotInfoMap[strconv.Itoa(i)].NodeId = strconv.Itoa(newNodeId)
newSlotInfoMap[strconv.Itoa(i)].SrcNodeId = strconv.Itoa(oldNodeId)
newSlotInfoMap[strconv.Itoa(i)].TargetNodeId = strconv.Itoa(newNodeId)
//update the new slotinfo to zk
jsonStr, err := utils.ToJson(newSlotInfoMap[strconv.Itoa(i)])
if err != nil {
log.Errorf("Can not convert to json string from obj [%s]", newSlotInfoMap[strconv.Itoa(i)])
} else {
_, err = zkHelper.CoverCreate("/yundis/ids/"+strconv.Itoa(i), []byte(jsonStr), 0, zk.WorldACL(zk.PermAll))
if err != nil {
log.Errorf("Change the value of /yundis/ids/%d fail, err:%s.", i, err)
}
//zkHelper.Set("/yundis/ids/"+strconv.Itoa(i), []byte(jsonStr), 1)
}
}
}
if isChanged {
log.Info("Update the slotinfoMaps.")
//slotinfoMaps.SetSlotInfoMap(infoMap)
slotinfoMaps.SetSlotInfoMap(newSlotInfoMap)
}
}
/**
* generate the data of node by map
*/
func (self *SlotAllocation) ToNodeData() ([]byte, error) {
str, err := utils.ToJson(self)
if err != nil {
return []byte{}, err
}
return []byte(str), nil
}
/**
* register it self to zk
* path : /yundis/nodes/{id}
*/
func (self *YundisServer) registerToZk() {
if !self.zkHelper.PathExist("/yundis/nodes") {
self.zkHelper.RecCreatePathNx("/yundis/nodes")
}
//register itself to zk
nodeinfo := NodeInfo{
Id: self.Id,
Host: self.Host,
Port: self.Port,
RedisHost: self.RedisHost,
RedisPort: self.RedisPort,
}
jsonStr, err := utils.ToJson(nodeinfo)
if err != nil {
log.Errorf("Convert to json string error:%s", err)
panic(err)
}
_, err = self.zkHelper.CoverCreate("/yundis/nodes/"+strconv.Itoa(self.Id), []byte(jsonStr), zk.FlagEphemeral, zk.WorldACL(zk.PermAll))
if err != nil {
log.Errorf("register node error:%s", err)
panic(err)
}
}
/**
* load the slot's info to map.
*/
func (self *SlotInfoMaps) GetSlotInfoMapFromZk() map[string]*SlotInfo {
log.Info("Read the slot's info from zk.")
if !self.zk.PathExist("/yundis/ids") {
_, err := self.zk.Create("/yundis/ids", []byte{}, 0, zk.WorldACL(zk.PermAll))
log.Errorf("can not create path %s, err: %s", "/yundis/ids", err)
}
infoMap := make(map[string]*SlotInfo)
for i := 0; i < self.slotCount; i++ {
strI := strconv.Itoa(i)
bytes, _, err := self.zk.Get("/yundis/ids/" + strI)
if err != nil || len(bytes) == 0 {
slotInfo := &SlotInfo{
SlotId: strI,
State: SlotStateNormal,
}
infoMap[strI] = slotInfo
dataStr, err := utils.ToJson(slotInfo)
if err != nil {
log.Errorf("Can not convert %s to json. err:%s", slotInfo, err)
continue
}
self.zk.Create("/yundis/ids/"+strI, []byte(dataStr), 0, zk.WorldACL(zk.PermAll))
} else {
log.Infof("Read the data form path %s", "/yundis/ids/"+strI)
var slotInfo SlotInfo
err = utils.JsonParse(string(bytes), &slotInfo)
if err != nil {
log.Errorf("Can not parse data from node %s, err: %s", "/yundis/ids/"+strI, err)
continue
} else {
infoMap[strI] = &slotInfo
}
}
}
return infoMap
}
/**
* 1. for each the slotinfo list
*/
func (self *NodeInfoMaps) ModifySlotState(newInfoMap map[string]*NodeInfo) {
log.Info("Call modify slotinfo state.")
// get the slotinfo map
slotInfoMap := self.slotInfoMaps.CloneSlotInfoMap()
var changedSlots []*SlotInfo
for slotId, slotInfo := range slotInfoMap {
log.Debugf("Orginal SlotInfo [SlotId:%s, State:%s, NodeId:%s, SrcNodeId:%s, TargetNodeId:%s]", slotInfo.SlotId, slotInfo.State, slotInfo.NodeId, slotInfo.SrcNodeId, slotInfo.TargetNodeId)
isChanged := false
if _, ok := newInfoMap[slotInfo.NodeId]; ok { //the node alive
if slotInfo.State == SlotStateDead { //active this slot
slotInfo.State = SlotStateNormal
isChanged = true
log.Infof("Active slot:%s", slotId)
}
} else { // the node dead
if slotInfo.State == SlotStateNormal { //inactive this slot
slotInfo.State = SlotStateDead
isChanged = true
log.Infof("Inactive slot:%s", slotId)
}
}
if isChanged {
changedSlots = append(changedSlots, slotInfo)
log.Debugf("SlotInfo after changed [SlotId:%s, State:%s, NodeId:%s, SrcNodeId:%s, TargetNodeId:%s]", slotInfo.SlotId, slotInfo.State, slotInfo.NodeId, slotInfo.SrcNodeId, slotInfo.TargetNodeId)
}
}
//change current slotInfoMap
self.slotInfoMaps.SetSlotInfoMap(slotInfoMap)
if len(changedSlots) > 0 { //wether need to sync the value to zk.
log.Info("Update the zk by new slotinfo map.")
//get the zk locker
zkLocker := self.zk.GetLocker("/yundis/idslocker")
err := zkLocker.Lock()
if err != nil {
log.Warnf("Can not lock the idslocker, err:%s", err)
} else {
log.Info("Get the latest slotinfo map from zk before update them.")
latestSlotMap := self.slotInfoMaps.GetSlotInfoMapFromZk()
//update the slot to zk
var changedSlotIds []int
for _, slotInfo := range changedSlots {
if slotInfo.State == latestSlotMap[slotInfo.SlotId].State {
continue
}
log.Infof("Begin to update the slotinfo[%s] to zk.", slotInfo.SlotId)
//record the id
slotId, err := strconv.Atoi(slotInfo.SlotId)
if err == nil {
changedSlotIds = append(changedSlotIds, slotId)
} else {
log.Errorf("Convert slotId to int from string. err:%s", err)
}
//update the slot value
slotJsonStr, err := utils.ToJson(slotInfo)
if err == nil {
_, err = self.zk.Set("/yundis/ids/"+slotInfo.SlotId, []byte(slotJsonStr))
if err != nil {
log.Errorf("Can not set the value of /yundis/ids/"+slotInfo.SlotId+" to zk. err : %s", err)
}
} else {
log.Errorf("Can not convert %+v to jsonstr. err : %s", slotInfo, err)
}
}
//update the value of /yundis/ids
if len(changedSlotIds) > 0 {
sort.Ints(changedSlotIds) //sort the array.
log.Infof("update the %s to /yundis/ids.", changedSlotIds)
idsJsonStr, err := utils.ToJson(changedSlotIds)
if err == nil {
_, err = self.zk.Set("/yundis/ids", []byte(idsJsonStr))
if err != nil {
log.Errorf("Can not set the value of /yundis/ids. err : %s", err)
}
} else {
log.Errorf("Can not convert %+v to jsonstr. err:%s", changedSlotIds, err)
}
}
//Unlock
err := zkLocker.Unlock()
if err != nil {
log.Errorf("Unlock the idslocker error. err:%s", err)
}
}
} else {
log.Info("Do not need to refresh slotinfo to zk.")
}
}
func (self *AdminCmdHandler) DoReblance() bool {
log.Info("Do reblance.")
//get the allocation from zk.
allocationsFromZk := GetSlotAllocationsFromZk(self.yundis.zkHelper)
if allocationsFromZk != nil {
//get the node list
nodeInfoMaps := self.yundis.nodeinfoMaps.getNodeInfoMapFromZk()
self.yundis.nodeinfoMaps.SetNodeInfoMap(nodeInfoMaps) //update the nodeinfo map
nodeSlotsMap := allocationsFromZk.getNodeMap()
var addedNodeIds []int
for _, nodeInfo := range nodeInfoMaps {
if _, ok := nodeSlotsMap[nodeInfo.Id]; ok {
} else {
addedNodeIds = append(addedNodeIds, nodeInfo.Id)
}
}
if len(addedNodeIds) > 0 {
log.Info("New Added node list : ", addedNodeIds)
for _, nodeId := range addedNodeIds {
log.Infof("Add node %d to allocations.", nodeId)
err := allocationsFromZk.AddNode(nodeId)
if err != nil {
log.Errorf("Error when add node %d to allocation. err : %s", nodeId, err)
}
}
zkLocker := self.yundis.zkHelper.GetLocker("/yundis/idslocker")
lockErr := zkLocker.Lock()
if lockErr != nil {
log.Errorf("Can not lock /yundis/idslocker. err : %s", lockErr)
return false
} else {
//current slot map
slotMap := self.yundis.slotinfoMaps.GetSlotInfoMapFromZk()
var affectedSlots []*SlotInfo
//for each slot:node map
for slotId, nodeId := range allocationsFromZk.Allocations {
slotInfo := slotMap[slotId]
if slotInfo != nil && slotInfo.NodeId != strconv.Itoa(nodeId) {
//modify the slot's migrate state
slotInfo.MigrateState = MigStateMigrating
slotInfo.SrcNodeId = slotInfo.NodeId
slotInfo.NodeId = strconv.Itoa(nodeId)
slotInfo.TargetNodeId = strconv.Itoa(nodeId)
affectedSlots = append(affectedSlots, slotInfo)
}
}
self.yundis.slotinfoMaps.SetSlotInfoMap(slotMap) //update local slot map.
//update allocation to zk
log.Info("Update the allocation after changed to zk.")
err := SyncToZk(allocationsFromZk, self.yundis.zkHelper)
if err != nil {
log.Errorf("Can not sync allocations to zk.err : %s", err)
}
//update slot info to zk
if len(affectedSlots) > 0 {
log.Info("Update slotInfo to zk.")
var affectedSlotIds []int
for _, slotInfo := range affectedSlots {
log.Infof("Begin to update the slotinfo[%s] to zk.", slotInfo.SlotId)
//record the id
slotId, err := strconv.Atoi(slotInfo.SlotId)
if err == nil {
affectedSlotIds = append(affectedSlotIds, slotId)
} else {
log.Errorf("Convert slotId to int from string. err:%s", err)
}
//update the slot value
slotJsonStr, err := utils.ToJson(slotInfo)
if err == nil {
_, err = self.yundis.zkHelper.Set("/yundis/ids/"+slotInfo.SlotId, []byte(slotJsonStr))
if err != nil {
log.Errorf("Can not set the value of /yundis/ids/"+slotInfo.SlotId+" to zk. err : %s", err)
}
} else {
log.Errorf("Can not convert %+v to jsonstr. err : %s", slotInfo, err)
}
}
//update the value of /yundis/ids
if len(affectedSlotIds) > 0 {
sort.Ints(affectedSlotIds) //sort the array.
log.Infof("update the %s to /yundis/ids.", affectedSlotIds)
idsJsonStr, err := utils.ToJson(affectedSlotIds)
if err == nil {
_, err = self.yundis.zkHelper.Set("/yundis/ids", []byte(idsJsonStr))
if err != nil {
log.Errorf("Can not set the value of /yundis/ids. err : %s", err)
}
} else {
log.Errorf("Can not convert %+v to jsonstr. err:%s", affectedSlotIds, err)
}
}
}
lockErr = zkLocker.Unlock()
if lockErr != nil {
log.Errorf("Can not release lock /yundis/dislocker. err : %s", lockErr)
}
}
}
return true
} else {
log.Error("When do reblance, can not reload allocations from zk.")
}
return false
}