Example #1
0
// processResponse reads one health check response, and notifies HealthCheckStatsListener.
// It returns bool to indicate if the caller should reconnect. We do not need to reconnect when the streaming is working.
func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, endPoint *topodatapb.EndPoint, stream tabletconn.StreamHealthReader) (bool, error) {
	select {
	case <-hcc.ctx.Done():
		return false, hcc.ctx.Err()
	default:
	}

	shr, err := stream.Recv()
	if err != nil {
		return true, err
	}
	// TODO(liang): remove after bug fix in tablet.
	if shr.Target == nil || shr.RealtimeStats == nil {
		return false, fmt.Errorf("health stats is not valid: %v", shr)
	}

	// an app-level error from tablet, force serving state.
	var healthErr error
	serving := shr.Serving
	if shr.RealtimeStats.HealthError != "" {
		healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError)
		serving = false
	}

	if hcc.target.TabletType == topodatapb.TabletType_UNKNOWN {
		// The first time we see response for the endpoint.
		hcc.update(shr, serving, healthErr, true)
		hc.mu.Lock()
		hc.addEndPointToTargetProtected(hcc.target, endPoint)
		hc.mu.Unlock()
	} else if hcc.target.TabletType != shr.Target.TabletType {
		// tablet type changed for the tablet
		log.Infof("HealthCheckUpdate(Type Change): %v, EP: %v/%+v, target %+v => %+v, reparent time: %v", hcc.name, hcc.cell, endPoint, hcc.target, shr.Target, shr.TabletExternallyReparentedTimestamp)
		hc.mu.Lock()
		hc.deleteEndPointFromTargetProtected(hcc.target, endPoint)
		hcc.update(shr, serving, healthErr, true)
		hc.addEndPointToTargetProtected(shr.Target, endPoint)
		hc.mu.Unlock()
	} else {
		hcc.update(shr, serving, healthErr, false)
	}
	// notify downstream for tablettype and realtimestats change
	if hc.listener != nil {
		hcc.mu.RLock()
		eps := &EndPointStats{
			EndPoint: endPoint,
			Cell:     hcc.cell,
			Name:     hcc.name,
			Target:   hcc.target,
			Up:       hcc.up,
			Serving:  hcc.serving,
			Stats:    hcc.stats,
			TabletExternallyReparentedTimestamp: hcc.tabletExternallyReparentedTimestamp,
			LastError:                           hcc.lastError,
		}
		hcc.mu.RUnlock()
		hc.listener.StatsUpdate(eps)
	}
	return false, nil
}
Example #2
0
// processResponse reads one health check response, and notifies HealthCheckStatsListener.
// It returns bool to indicate if the caller should reconnect. We do not need to reconnect when the streaming is working.
func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, stream tabletconn.StreamHealthReader) (bool, error) {
	select {
	case <-hcc.ctx.Done():
		return false, hcc.ctx.Err()
	default:
	}

	shr, err := stream.Recv()
	if err != nil {
		return true, err
	}

	// Check for invalid data, better than panicking.
	if shr.Target == nil || shr.RealtimeStats == nil {
		return false, fmt.Errorf("health stats is not valid: %v", shr)
	}

	// an app-level error from tablet, force serving state.
	var healthErr error
	serving := shr.Serving
	if shr.RealtimeStats.HealthError != "" {
		healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError)
		serving = false
	}

	if hcc.tabletStats.Target.TabletType == topodatapb.TabletType_UNKNOWN {
		// The first time we see response for the tablet.
		hcc.update(shr, serving, healthErr)
		hc.mu.Lock()
		hc.addTabletToTargetProtected(hcc.tabletStats.Target, hcc.tabletStats.Tablet)
		hc.mu.Unlock()
	} else if hcc.tabletStats.Target.TabletType != shr.Target.TabletType {
		// tablet type changed for the tablet
		log.Infof("HealthCheckUpdate(Type Change): %v, tablet: %v/%+v, target %+v => %+v, reparent time: %v", hcc.tabletStats.Name, hcc.tabletStats.Tablet.Alias.Cell, hcc.tabletStats.Tablet, hcc.tabletStats.Target, shr.Target, shr.TabletExternallyReparentedTimestamp)
		hc.mu.Lock()
		hc.deleteTabletFromTargetProtected(hcc.tabletStats.Target, hcc.tabletStats.Tablet)
		hcc.update(shr, serving, healthErr)
		hc.addTabletToTargetProtected(shr.Target, hcc.tabletStats.Tablet)
		hc.mu.Unlock()
	} else {
		hcc.update(shr, serving, healthErr)
	}
	// notify downstream for tablettype and realtimestats change
	if hc.listener != nil {
		hcc.mu.RLock()
		ts := hcc.tabletStats
		hcc.mu.RUnlock()
		hc.listener.StatsUpdate(&ts)
	}
	return false, nil
}
Example #3
0
// processResponse reads one health check response, and notifies HealthCheckStatsListener.
// It returns bool to indicate if the caller should reconnect. We do not need to reconnect when the streaming is working.
func (hcc *healthCheckConn) processResponse(hc *HealthCheckImpl, stream tabletconn.StreamHealthReader) (bool, error) {
	select {
	case <-hcc.ctx.Done():
		return false, hcc.ctx.Err()
	default:
	}

	shr, err := stream.Recv()
	if err != nil {
		return true, err
	}

	// Check for invalid data, better than panicking.
	if shr.Target == nil || shr.RealtimeStats == nil {
		return false, fmt.Errorf("health stats is not valid: %v", shr)
	}

	// an app-level error from tablet, force serving state.
	var healthErr error
	serving := shr.Serving
	if shr.RealtimeStats.HealthError != "" {
		healthErr = fmt.Errorf("vttablet error: %v", shr.RealtimeStats.HealthError)
		serving = false
	}

	// In the case where a tablet changes type (but not for the
	// initial message), we want to log it, and maybe advertise it too.
	if hcc.tabletStats.Target.TabletType != topodatapb.TabletType_UNKNOWN && hcc.tabletStats.Target.TabletType != shr.Target.TabletType {
		// The Tablet type changed for the tablet. Get old value.
		hcc.mu.RLock()
		oldTs := hcc.tabletStats
		hcc.mu.RUnlock()

		// Log and maybe notify
		log.Infof("HealthCheckUpdate(Type Change): %v, tablet: %v/%+v, target %+v => %+v, reparent time: %v", oldTs.Name, oldTs.Tablet.Alias.Cell, oldTs.Tablet, oldTs.Target, shr.Target, shr.TabletExternallyReparentedTimestamp)
		if hc.listener != nil && hc.sendDownEvents {
			oldTs.Up = false
			hc.listener.StatsUpdate(&oldTs)
		}
	}

	// Update our record, and notify downstream for tabletType and
	// realtimeStats change.
	ts := hcc.update(shr, serving, healthErr)
	if hc.listener != nil {
		hc.listener.StatsUpdate(&ts)
	}
	return false, nil
}