func (top *Topology) Close(proxyName string) {
// delete fence znode
pi, err := models.GetProxyInfo(top.zkConn, top.ProductName, proxyName)
if err != nil {
log.Errorf("killing fence error, proxy %s is not exists", proxyName)
} else {
zkhelper.DeleteRecursive(top.zkConn, path.Join(models.GetProxyFencePath(top.ProductName), pi.Addr), -1)
}
// delete ephemeral znode
zkhelper.DeleteRecursive(top.zkConn, path.Join(models.GetProxyPath(top.ProductName), proxyName), -1)
top.zkConn.Close()
}
func WaitForReceiverWithTimeout(zkConn zkhelper.Conn, productName string, actionZkPath string, proxies []ProxyInfo, timeoutInMs int) error {
if len(proxies) == 0 {
return nil
}
times := 0
proxyIds := make(map[string]bool)
for _, p := range proxies {
proxyIds[p.Id] = true
}
// check every 500ms
for times < timeoutInMs/500 {
if times >= 6 && (times*500)%1000 == 0 {
log.Warnf("abnormal waiting time for receivers: %s %v", actionZkPath, proxyIds)
}
// get confirm ids
nodes, _, err := zkConn.Children(actionZkPath)
if err != nil {
return errors.Trace(err)
}
for _, node := range nodes {
id := path.Base(node)
delete(proxyIds, id)
}
if len(proxyIds) == 0 {
return nil
}
times++
time.Sleep(500 * time.Millisecond)
}
log.Warn("proxies didn't responed: ", proxyIds)
// set offline proxies
for id, _ := range proxyIds {
log.Errorf("mark proxy %s to PROXY_STATE_MARK_OFFLINE", id)
if err := SetProxyStatus(zkConn, productName, id, PROXY_STATE_MARK_OFFLINE); err != nil {
return errors.Trace(err)
}
}
return ErrReceiverTimeout
}
func callApi(method HttpMethod, apiPath string, params interface{}, retVal interface{}) error {
if apiPath[0] != '/' {
return errors.Errorf("api path must starts with /")
}
url := "http://" + globalEnv.DashboardAddr() + apiPath
client := &http.Client{Transport: http.DefaultTransport}
b, err := json.Marshal(params)
if err != nil {
return errors.Trace(err)
}
req, err := http.NewRequest(string(method), url, strings.NewReader(string(b)))
if err != nil {
return errors.Trace(err)
}
resp, err := client.Do(req)
if err != nil {
log.Errorf("can't connect to dashboard, please check 'dashboard_addr' is corrent in config file")
return errors.Trace(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Trace(err)
}
if resp.StatusCode == 200 {
err := json.Unmarshal(body, retVal)
if err != nil {
return errors.Trace(err)
}
return nil
}
return errors.Errorf("http status code %d, %s", resp.StatusCode, string(body))
}
func (t *MigrateTask) migrateSingleSlot(slotId int, to int) error {
// set slot status
s, err := models.GetSlot(t.zkConn, t.productName, slotId)
if err != nil {
log.ErrorErrorf(err, "get slot info failed")
return err
}
if s.State.Status == models.SLOT_STATUS_OFFLINE {
log.Warnf("status is offline: %+v", s)
return nil
}
from := s.GroupId
if s.State.Status == models.SLOT_STATUS_MIGRATE {
from = s.State.MigrateStatus.From
}
// make sure from group & target group exists
exists, err := models.GroupExists(t.zkConn, t.productName, from)
if err != nil {
return errors.Trace(err)
}
if !exists {
log.Errorf("src group %d not exist when migrate from %d to %d", from, from, to)
return errors.Errorf("group %d not found", from)
}
exists, err = models.GroupExists(t.zkConn, t.productName, to)
if err != nil {
return errors.Trace(err)
}
if !exists {
return errors.Errorf("group %d not found", to)
}
// cannot migrate to itself, just ignore
if from == to {
log.Warnf("from == to, ignore: %+v", s)
return nil
}
// modify slot status
if err := s.SetMigrateStatus(t.zkConn, from, to); err != nil {
log.ErrorErrorf(err, "set migrate status failed")
return err
}
err = t.Migrate(s, from, to, func(p SlotMigrateProgress) {
// on migrate slot progress
if p.Remain%5000 == 0 {
log.Infof("%+v", p)
}
})
if err != nil {
log.ErrorErrorf(err, "migrate slot failed")
return err
}
// migrate done, change slot status back
s.State.Status = models.SLOT_STATUS_ONLINE
s.State.MigrateStatus.From = models.INVALID_ID
s.State.MigrateStatus.To = models.INVALID_ID
if err := s.Update(t.zkConn); err != nil {
log.ErrorErrorf(err, "update zk status failed, should be: %+v", s)
return err
}
return nil
}