func (mrl *mysqlReplicationLag) Report(typ topo.TabletType) (status map[string]string, err error) {
if !topo.IsSlaveType(typ) {
return nil, nil
}
rp, err := mrl.mysqld.SlaveStatus()
if err != nil {
return nil, err
}
if int(rp.SecondsBehindMaster) > mrl.allowedLagInSeconds {
return map[string]string{health.ReplicationLag: health.ReplicationLagHigh}, nil
}
return nil, nil
}
// MySQLReplication lag returns a reporter that reports the MySQL
// replication lag. It uses the key "replication_lag".
func MySQLReplicationLag(mysqld *Mysqld, allowedLagInSeconds int) health.Reporter {
return health.FunctionReporter(func(typ topo.TabletType) (map[string]string, error) {
if !topo.IsSlaveType(typ) {
return nil, nil
}
rp, err := mysqld.SlaveStatus()
if err != nil {
return nil, err
}
if int(rp.SecondsBehindMaster) > allowedLagInSeconds {
return map[string]string{health.ReplicationLag: health.ReplicationLagHigh}, nil
}
return nil, nil
})
}
// runHealthCheck takes the action mutex, runs the health check,
// and if we need to change our state, do it.
// If we are the master, we don't change our type, healthy or not.
// If we are not the master, we change to spare if not healthy,
// or to the passed in targetTabletType if healthy.
//
// Note we only update the topo record if we need to, that is if our type or
// health details changed.
func (agent *ActionAgent) runHealthCheck(targetTabletType pbt.TabletType) {
agent.actionMutex.Lock()
defer agent.actionMutex.Unlock()
// read the current tablet record and tablet control
agent.mutex.Lock()
tablet := agent._tablet
tabletControl := agent._tabletControl
agent.mutex.Unlock()
// figure out if we should be running the query service
shouldQueryServiceBeRunning := false
var blacklistedTables []string
if topo.IsRunningQueryService(targetTabletType) && !agent.BinlogPlayerMap.isRunningFilteredReplication() {
shouldQueryServiceBeRunning = true
if tabletControl != nil {
blacklistedTables = tabletControl.BlacklistedTables
if tabletControl.DisableQueryService {
shouldQueryServiceBeRunning = false
}
}
}
// run the health check
typeForHealthCheck := targetTabletType
if tablet.Type == pbt.TabletType_MASTER {
typeForHealthCheck = pbt.TabletType_MASTER
}
replicationDelay, err := agent.HealthReporter.Report(topo.IsSlaveType(typeForHealthCheck), shouldQueryServiceBeRunning)
health := make(map[string]string)
if err == nil {
if replicationDelay > *unhealthyThreshold {
err = fmt.Errorf("reported replication lag: %v higher than unhealthy threshold: %v", replicationDelay.Seconds(), unhealthyThreshold.Seconds())
} else if replicationDelay > *degradedThreshold {
health[topo.ReplicationLag] = topo.ReplicationLagHigh
}
}
agent.lastHealthMapCount.Set(int64(len(health)))
// Figure out if we should be running QueryService, see if we are,
// and reconcile.
if err != nil {
if tablet.Type != pbt.TabletType_WORKER {
// We are not healthy and must shut down QueryService.
// At the moment, the only exception to this are "worker" tablets which
// still must serve queries e.g. as source tablet during a "SplitClone".
shouldQueryServiceBeRunning = false
}
}
isQueryServiceRunning := agent.QueryServiceControl.IsServing()
if shouldQueryServiceBeRunning {
if !isQueryServiceRunning {
// send the type we want to be, not the type we are
currentType := tablet.Type
if tablet.Type == pbt.TabletType_SPARE {
tablet.Type = targetTabletType
}
// we remember this new possible error
err = agent.allowQueries(tablet.Tablet, blacklistedTables)
// restore the current type
tablet.Type = currentType
}
} else {
if isQueryServiceRunning {
// we are not healthy or should not be running the
// query service, shut it down.
agent.stopQueryService()
}
}
// save the health record
record := &HealthRecord{
Error: err,
ReplicationDelay: replicationDelay,
Time: time.Now(),
}
agent.History.Add(record)
// try to figure out the mysql port if we don't have it yet
if _, ok := tablet.PortMap["mysql"]; !ok {
// we don't know the port, try to get it from mysqld
mysqlPort, err := agent.MysqlDaemon.GetMysqlPort()
if err != nil {
// Don't log if we're already in a waiting-for-mysql state.
agent.mutex.Lock()
if !agent._waitingForMysql {
log.Warningf("Can't get mysql port, won't populate Tablet record in topology (will retry silently at healthcheck interval %v): %v", *healthCheckInterval, err)
agent._waitingForMysql = true
}
agent.mutex.Unlock()
} else {
log.Infof("Updating tablet mysql port to %v", mysqlPort)
if err := agent.TopoServer.UpdateTabletFields(agent.batchCtx, tablet.Alias, func(tablet *pbt.Tablet) error {
tablet.PortMap["mysql"] = int32(mysqlPort)
return nil
}); err != nil {
log.Infof("Error updating mysql port in tablet record: %v", err)
return
}
// save the port so we don't update it again next time
// we do the health check.
agent.mutex.Lock()
agent._tablet.PortMap["mysql"] = int32(mysqlPort)
agent._waitingForMysql = false
agent.mutex.Unlock()
}
}
// remember our health status
agent.mutex.Lock()
agent._healthy = err
agent._healthyTime = time.Now()
agent._replicationDelay = replicationDelay
terTime := agent._tabletExternallyReparentedTime
agent.mutex.Unlock()
// send it to our observers
// (the Target has already been updated when restarting the
// query service earlier)
// FIXME(alainjobart,liguo) add CpuUsage
stats := &pb.RealtimeStats{
SecondsBehindMaster: uint32(replicationDelay.Seconds()),
}
stats.SecondsBehindMasterFilteredReplication, stats.BinlogPlayersCount = agent.BinlogPlayerMap.StatusSummary()
if err != nil {
stats.HealthError = err.Error()
}
defer func() {
var ts int64
if !terTime.IsZero() {
ts = terTime.Unix()
}
agent.QueryServiceControl.BroadcastHealth(ts, stats)
}()
// Update our topo.Server state, start with no change
newTabletType := tablet.Type
if err != nil {
// The tablet is not healthy, let's see what we need to do
if tablet.Type != targetTabletType {
if tablet.Type != pbt.TabletType_SPARE {
// we only log if we're not in spare,
// as the spare state is normal for a
// failed health check.
log.Infof("Tablet not healthy and in state %v, not changing it: %v", tablet.Type, err)
}
return
}
// Note that if the query service is running, we may
// need to stop it. The post-action callback will do
// it, and it will be done after we change our state,
// so it's the right order, let it do it.
log.Infof("Tablet not healthy, converting it from %v to spare: %v", targetTabletType, err)
newTabletType = pbt.TabletType_SPARE
} else {
// We are healthy, maybe with health, see if we need
// to update the record. We only change from spare to
// our target type.
if tablet.Type == pbt.TabletType_SPARE {
newTabletType = targetTabletType
}
if tablet.Type == newTabletType && topo.IsHealthEqual(health, tablet.HealthMap) {
// no change in health, not logging anything,
// and we're done
return
}
// we need to update our state
log.Infof("Updating tablet record as healthy type %v -> %v with health details %v -> %v", tablet.Type, newTabletType, tablet.HealthMap, health)
}
// Change the Type, update the health. Note we pass in a map
// that's not nil, meaning if it's empty, we will clear it.
if err := topotools.ChangeType(agent.batchCtx, agent.TopoServer, tablet.Alias, newTabletType, health); err != nil {
log.Infof("Error updating tablet record: %v", err)
return
}
tablet.HealthMap = health
tablet.Type = newTabletType
// Rebuild the serving graph in our cell, only if we're dealing with
// a serving type
if err := agent.updateServingGraph(tablet, targetTabletType); err != nil {
log.Warningf("updateServingGraph failed (will still run post action callbacks, serving graph might be out of date): %v", err)
}
// run the post action callbacks, not much we can do with returned error
if err := agent.refreshTablet(agent.batchCtx, "healthcheck"); err != nil {
log.Warningf("refreshTablet failed: %v", err)
}
}
func (ta *TabletActor) multiRestore(actionNode *actionnode.ActionNode) (err error) {
args := actionNode.Args.(*actionnode.MultiRestoreArgs)
// read our current tablet, verify its state
// we only support restoring to the master or active replicas
tablet, err := ta.ts.GetTablet(ta.tabletAlias)
if err != nil {
return err
}
if tablet.Type != topo.TYPE_MASTER && !topo.IsSlaveType(tablet.Type) {
return fmt.Errorf("expected master, or slave type, not %v: %v", tablet.Type, ta.tabletAlias)
}
// get source tablets addresses
sourceAddrs := make([]*url.URL, len(args.SrcTabletAliases))
keyRanges := make([]key.KeyRange, len(args.SrcTabletAliases))
fromStoragePaths := make([]string, len(args.SrcTabletAliases))
for i, alias := range args.SrcTabletAliases {
t, e := ta.ts.GetTablet(alias)
if e != nil {
return e
}
sourceAddrs[i] = &url.URL{
Host: t.Addr(),
Path: "/" + t.DbName(),
}
keyRanges[i], e = key.KeyRangesOverlap(tablet.KeyRange, t.KeyRange)
if e != nil {
return e
}
fromStoragePaths[i] = path.Join(ta.mysqld.SnapshotDir, "from-storage", fmt.Sprintf("from-%v-%v", keyRanges[i].Start.Hex(), keyRanges[i].End.Hex()))
}
// change type to restore, no change to replication graph
originalType := tablet.Type
tablet.Type = topo.TYPE_RESTORE
err = topo.UpdateTablet(ta.ts, tablet)
if err != nil {
return err
}
// first try to get the data from a remote storage
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for i, alias := range args.SrcTabletAliases {
wg.Add(1)
go func(i int, alias topo.TabletAlias) {
defer wg.Done()
h := hook.NewSimpleHook("copy_snapshot_from_storage")
h.ExtraEnv = make(map[string]string)
for k, v := range ta.hookExtraEnv() {
h.ExtraEnv[k] = v
}
h.ExtraEnv["KEYRANGE"] = fmt.Sprintf("%v-%v", keyRanges[i].Start.Hex(), keyRanges[i].End.Hex())
h.ExtraEnv["SNAPSHOT_PATH"] = fromStoragePaths[i]
h.ExtraEnv["SOURCE_TABLET_ALIAS"] = alias.String()
hr := h.Execute()
if hr.ExitStatus != hook.HOOK_SUCCESS {
rec.RecordError(fmt.Errorf("%v hook failed(%v): %v", h.Name, hr.ExitStatus, hr.Stderr))
}
}(i, alias)
}
wg.Wait()
// run the action, scrap if it fails
if rec.HasErrors() {
log.Infof("Got errors trying to get snapshots from storage, trying to get them from original tablets: %v", rec.Error())
err = ta.mysqld.MultiRestore(tablet.DbName(), keyRanges, sourceAddrs, nil, args.Concurrency, args.FetchConcurrency, args.InsertTableConcurrency, args.FetchRetryCount, args.Strategy)
} else {
log.Infof("Got snapshots from storage, reading them from disk directly")
err = ta.mysqld.MultiRestore(tablet.DbName(), keyRanges, nil, fromStoragePaths, args.Concurrency, args.FetchConcurrency, args.InsertTableConcurrency, args.FetchRetryCount, args.Strategy)
}
if err != nil {
if e := topotools.Scrap(ta.ts, ta.tabletAlias, false); e != nil {
log.Errorf("Failed to Scrap after failed RestoreFromMultiSnapshot: %v", e)
}
return err
}
// restore type back
tablet.Type = originalType
return topo.UpdateTablet(ta.ts, tablet)
}
// runHealthCheck takes the action mutex, runs the health check,
// and if we need to change our state, do it.
// If we are the master, we don't change our type, healthy or not.
// If we are not the master, we change to spare if not healthy,
// or to the passed in targetTabletType if healthy.
//
// Note we only update the topo record if we need to, that is if our type or
// health details changed.
func (agent *ActionAgent) runHealthCheck(targetTabletType topo.TabletType) {
agent.actionMutex.Lock()
defer agent.actionMutex.Unlock()
// read the current tablet record and tablet control
agent.mutex.Lock()
tablet := agent._tablet
tabletControl := agent._tabletControl
agent.mutex.Unlock()
// figure out if we should be running the query service
shouldQueryServiceBeRunning := false
var blacklistedTables []string
if topo.IsRunningQueryService(targetTabletType) && agent.BinlogPlayerMap.size() == 0 {
shouldQueryServiceBeRunning = true
if tabletControl != nil {
blacklistedTables = tabletControl.BlacklistedTables
if tabletControl.DisableQueryService {
shouldQueryServiceBeRunning = false
}
}
}
// run the health check
typeForHealthCheck := targetTabletType
if tablet.Type == topo.TYPE_MASTER {
typeForHealthCheck = topo.TYPE_MASTER
}
replicationDelay, err := agent.HealthReporter.Report(topo.IsSlaveType(typeForHealthCheck), shouldQueryServiceBeRunning)
health := make(map[string]string)
if err == nil {
if replicationDelay > *unhealthyThreshold {
err = fmt.Errorf("reported replication lag: %v higher than unhealthy threshold: %v", replicationDelay.Seconds(), unhealthyThreshold.Seconds())
} else if replicationDelay > *degradedThreshold {
health[topo.ReplicationLag] = topo.ReplicationLagHigh
}
}
// Figure out if we should be running QueryService, see if we are,
// and reconcile.
if err != nil {
// we are not healthy, we should not be running QueryService
shouldQueryServiceBeRunning = false
}
isQueryServiceRunning := agent.QueryServiceControl.IsServing()
if shouldQueryServiceBeRunning {
if !isQueryServiceRunning {
// we remember this new possible error
err = agent.allowQueries(tablet.Tablet, blacklistedTables)
}
} else {
if isQueryServiceRunning {
// we are not healthy or should not be running the
// query service, shut it down.
agent.disallowQueries()
}
}
// save the health record
record := &HealthRecord{
Error: err,
ReplicationDelay: replicationDelay,
Time: time.Now(),
}
agent.History.Add(record)
// try to figure out the mysql port if we don't have it yet
if _, ok := tablet.Portmap["mysql"]; !ok {
// we don't know the port, try to get it from mysqld
mysqlPort, err := agent.MysqlDaemon.GetMysqlPort()
if err != nil {
// Don't log if we're already in a waiting-for-mysql state.
agent.mutex.Lock()
if !agent._waitingForMysql {
log.Warningf("Can't get mysql port, won't populate Tablet record in topology (will retry silently at healthcheck interval %v): %v", *healthCheckInterval, err)
agent._waitingForMysql = true
}
agent.mutex.Unlock()
} else {
log.Infof("Updating tablet mysql port to %v", mysqlPort)
if err := agent.TopoServer.UpdateTabletFields(agent.batchCtx, tablet.Alias, func(tablet *topo.Tablet) error {
tablet.Portmap["mysql"] = mysqlPort
return nil
}); err != nil {
log.Infof("Error updating mysql port in tablet record: %v", err)
return
}
// save the port so we don't update it again next time
// we do the health check.
agent.mutex.Lock()
agent._tablet.Portmap["mysql"] = mysqlPort
agent._waitingForMysql = false
agent.mutex.Unlock()
}
}
// remember our health status
agent.mutex.Lock()
agent._healthy = err
agent._healthyTime = time.Now()
agent._replicationDelay = replicationDelay
agent.mutex.Unlock()
// send it to our observers, after we've updated the tablet state
// (Tablet is a pointer, and below we will alter the Tablet
// record to be correct.
hsr := &actionnode.HealthStreamReply{
Tablet: tablet.Tablet,
BinlogPlayerMapSize: agent.BinlogPlayerMap.size(),
ReplicationDelay: replicationDelay,
}
if err != nil {
hsr.HealthError = err.Error()
}
defer agent.BroadcastHealthStreamReply(hsr)
// Update our topo.Server state, start with no change
newTabletType := tablet.Type
if err != nil {
// The tablet is not healthy, let's see what we need to do
if tablet.Type != targetTabletType {
if tablet.Type != topo.TYPE_SPARE {
// we only log if we're not in spare,
// as the spare state is normal for a
// failed health check.
log.Infof("Tablet not healthy and in state %v, not changing it: %v", tablet.Type, err)
}
return
}
// Note that if the query service is running, we may
// need to stop it. The post-action callback will do
// it, and it will be done after we change our state,
// so it's the right order, let it do it.
log.Infof("Tablet not healthy, converting it from %v to spare: %v", targetTabletType, err)
newTabletType = topo.TYPE_SPARE
} else {
// We are healthy, maybe with health, see if we need
// to update the record. We only change from spare to
// our target type.
if tablet.Type == topo.TYPE_SPARE {
newTabletType = targetTabletType
}
if tablet.Type == newTabletType && tablet.IsHealthEqual(health) {
// no change in health, not logging anything,
// and we're done
return
}
// we need to update our state
log.Infof("Updating tablet record as healthy type %v -> %v with health details %v -> %v", tablet.Type, newTabletType, tablet.Health, health)
agent.lastHealthMapCount.Set(int64(len(health)))
}
// Change the Type, update the health. Note we pass in a map
// that's not nil, meaning if it's empty, we will clear it.
if err := topotools.ChangeType(agent.batchCtx, agent.TopoServer, tablet.Alias, newTabletType, health); err != nil {
log.Infof("Error updating tablet record: %v", err)
return
}
tablet.Health = health
tablet.Type = newTabletType
// Rebuild the serving graph in our cell, only if we're dealing with
// a serving type
if err := agent.updateServingGraph(tablet, targetTabletType); err != nil {
log.Warningf("updateServingGraph failed (will still run post action callbacks, serving graph might be out of date): %v", err)
}
// run the post action callbacks, not much we can do with returned error
if err := agent.refreshTablet(agent.batchCtx, "healthcheck"); err != nil {
log.Warningf("refreshTablet failed: %v", err)
}
}