Exemple #1
0
func (conn *vtgateConn) ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*pbg.ExecuteEntityIdsRequest_EntityId, bindVars map[string]interface{}, tabletType pb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
	var s *proto.Session
	if session != nil {
		s = session.(*proto.Session)
	}
	request := proto.EntityIdsQuery{
		CallerID:          getEffectiveCallerID(ctx),
		Sql:               query,
		BindVariables:     bindVars,
		Keyspace:          keyspace,
		EntityColumnName:  entityColumnName,
		EntityKeyspaceIDs: proto.ProtoToEntityIds(entityKeyspaceIDs),
		TabletType:        topo.ProtoToTabletType(tabletType),
		Session:           s,
		NotInTransaction:  notInTransaction,
	}
	var result proto.QueryResult
	if err := conn.rpcConn.Call(ctx, "VTGate.ExecuteEntityIds", request, &result); err != nil {
		return nil, session, err
	}
	if err := vterrors.FromRPCError(result.Err); err != nil {
		return nil, result.Session, err
	}
	return result.Result, result.Session, nil
}
Exemple #2
0
func getKeyspaceShards(ctx context.Context, topoServ SrvTopoServer, cell, keyspace string, tabletType pb.TabletType) (string, *topo.SrvKeyspace, []topo.ShardReference, error) {
	srvKeyspace, err := topoServ.GetSrvKeyspace(ctx, cell, keyspace)
	if err != nil {
		return "", nil, nil, vterrors.NewVitessError(
			vtrpc.ErrorCode_INTERNAL_ERROR, err,
			"keyspace %v fetch error: %v", keyspace, err,
		)
	}

	// check if the keyspace has been redirected for this tabletType.
	tt := topo.ProtoToTabletType(tabletType)
	if servedFrom, ok := srvKeyspace.ServedFrom[tt]; ok {
		keyspace = servedFrom
		srvKeyspace, err = topoServ.GetSrvKeyspace(ctx, cell, keyspace)
		if err != nil {
			return "", nil, nil, vterrors.NewVitessError(
				vtrpc.ErrorCode_INTERNAL_ERROR, err,
				"keyspace %v fetch error: %v", keyspace, err,
			)
		}
	}

	partition, ok := srvKeyspace.Partitions[tt]
	if !ok {
		return "", nil, nil, vterrors.NewVitessError(
			vtrpc.ErrorCode_INTERNAL_ERROR, err,
			"No partition found for tabletType %v in keyspace %v", strings.ToLower(tabletType.String()), keyspace,
		)
	}
	return keyspace, srvKeyspace, partition.ShardReferences, nil
}
Exemple #3
0
// StreamExecute is part of the VTGateService interface
func (f *fakeVTGateService) StreamExecute(ctx context.Context, sql string, bindVariables map[string]interface{}, tabletType pb.TabletType, sendReply func(*proto.QueryResult) error) error {
	execCase, ok := execMap[sql]
	if !ok {
		return fmt.Errorf("no match for: %s", sql)
	}
	query := &proto.Query{
		Sql:           sql,
		BindVariables: bindVariables,
		TabletType:    topo.ProtoToTabletType(tabletType),
	}
	if !reflect.DeepEqual(query, execCase.execQuery) {
		return fmt.Errorf("request mismatch: got %+v, want %+v", query, execCase.execQuery)
	}
	if execCase.reply.Result != nil {
		result := proto.QueryResult{Result: &mproto.QueryResult{}}
		result.Result.Fields = execCase.reply.Result.Fields
		if err := sendReply(&result); err != nil {
			return err
		}
		for _, row := range execCase.reply.Result.Rows {
			result := proto.QueryResult{Result: &mproto.QueryResult{}}
			result.Result.Rows = [][]sqltypes.Value{row}
			if err := sendReply(&result); err != nil {
				return err
			}
		}
	}
	if execCase.reply.Error != "" {
		return errors.New(execCase.reply.Error)
	}
	return nil
}
Exemple #4
0
func (s *server) ChangeType(ctx context.Context, request *pb.ChangeTypeRequest) (*pb.ChangeTypeResponse, error) {
	ctx = callinfo.GRPCCallInfo(ctx)
	response := &pb.ChangeTypeResponse{}
	return response, s.agent.RPCWrapLockAction(ctx, actionnode.TabletActionChangeType, request, response, true, func() error {
		return s.agent.ChangeType(ctx, topo.ProtoToTabletType(request.TabletType))
	})
}
Exemple #5
0
// ExecuteBatchKeyspaceIds is the RPC version of
// vtgateservice.VTGateService method
func (vtg *VTGate) ExecuteBatchKeyspaceIds(ctx context.Context, request *pb.ExecuteBatchKeyspaceIdsRequest) (response *pb.ExecuteBatchKeyspaceIdsResponse, err error) {
	defer vtg.server.HandlePanic(&err)
	ctx = callerid.NewContext(callinfo.GRPCCallInfo(ctx),
		request.CallerId,
		callerid.NewImmediateCallerID("grpc client"))
	query := &proto.KeyspaceIdBatchQuery{
		Session:       proto.ProtoToSession(request.Session),
		Queries:       proto.ProtoToBoundKeyspaceIdQueries(request.Queries),
		TabletType:    topo.ProtoToTabletType(request.TabletType),
		AsTransaction: request.AsTransaction,
	}
	reply := new(proto.QueryResultList)
	executeErr := vtg.server.ExecuteBatchKeyspaceIds(ctx, query, reply)
	response = &pb.ExecuteBatchKeyspaceIdsResponse{
		Error: vtgate.VtGateErrorToVtRPCError(executeErr, reply.Error),
	}
	if executeErr == nil {
		response.Results = tproto.QueryResultListToProto3(reply.List)
		response.Session = proto.SessionToProto(reply.Session)
		return response, nil
	}
	if *vtgate.RPCErrorOnlyInReply {
		return response, nil
	}
	return nil, executeErr
}
Exemple #6
0
// ExecuteKeyspaceIds is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGate) ExecuteKeyspaceIds(ctx context.Context, request *pb.ExecuteKeyspaceIdsRequest) (response *pb.ExecuteKeyspaceIdsResponse, err error) {
	defer vtg.server.HandlePanic(&err)
	ctx = callerid.NewContext(callinfo.GRPCCallInfo(ctx),
		request.CallerId,
		callerid.NewImmediateCallerID("grpc client"))
	query := &proto.KeyspaceIdQuery{
		Sql:              string(request.Query.Sql),
		BindVariables:    tproto.Proto3ToBindVariables(request.Query.BindVariables),
		Keyspace:         request.Keyspace,
		KeyspaceIds:      key.ProtoToKeyspaceIds(request.KeyspaceIds),
		TabletType:       topo.ProtoToTabletType(request.TabletType),
		Session:          proto.ProtoToSession(request.Session),
		NotInTransaction: request.NotInTransaction,
	}
	reply := new(proto.QueryResult)
	executeErr := vtg.server.ExecuteKeyspaceIds(ctx, query, reply)
	response = &pb.ExecuteKeyspaceIdsResponse{
		Error: vtgate.VtGateErrorToVtRPCError(executeErr, reply.Error),
	}
	if executeErr == nil {
		response.Result = mproto.QueryResultToProto3(reply.Result)
		response.Session = proto.SessionToProto(reply.Session)
		return response, nil
	}
	if *vtgate.RPCErrorOnlyInReply {
		return response, nil
	}
	return nil, executeErr
}
Exemple #7
0
// ExecuteShards is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGate) ExecuteShards(ctx context.Context, request *pb.ExecuteShardsRequest) (response *pb.ExecuteShardsResponse, err error) {
	defer vtg.server.HandlePanic(&err)
	query := &proto.QueryShard{
		Sql:              string(request.Query.Sql),
		BindVariables:    tproto.Proto3ToBindVariables(request.Query.BindVariables),
		Keyspace:         request.Keyspace,
		Shards:           request.Shards,
		TabletType:       topo.ProtoToTabletType(request.TabletType),
		Session:          proto.ProtoToSession(request.Session),
		NotInTransaction: request.NotInTransaction,
	}
	reply := new(proto.QueryResult)
	executeErr := vtg.server.ExecuteShard(ctx, query, reply)
	response = &pb.ExecuteShardsResponse{
		Error: vtgate.VtGateErrorToVtRPCError(executeErr, reply.Error),
	}
	if executeErr == nil {
		response.Result = mproto.QueryResultToProto3(reply.Result)
		response.Session = proto.SessionToProto(reply.Session)
		return response, nil
	}
	if *vtgate.RPCErrorOnlyInReply {
		return response, nil
	}
	return nil, executeErr
}
Exemple #8
0
// Execute please see vtgateconn.Impl.Execute
func (conn *FakeVTGateConn) Execute(ctx context.Context, sql string, bindVars map[string]interface{}, tabletType pb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
	var s *proto.Session
	if session != nil {
		s = session.(*proto.Session)
	}
	query := &proto.Query{
		Sql:              sql,
		BindVariables:    bindVars,
		TabletType:       topo.ProtoToTabletType(tabletType),
		Session:          s,
		NotInTransaction: notInTransaction,
	}
	response, ok := conn.execMap[query.Sql]
	if !ok {
		return nil, nil, fmt.Errorf("no match for: %s", query.Sql)
	}
	if !reflect.DeepEqual(query, response.execQuery) {
		return nil, nil, fmt.Errorf(
			"Execute: %+v, want %+v", query, response.execQuery)
	}
	var reply mproto.QueryResult
	reply = *response.reply
	if s != nil {
		s = newSession(true, "test_keyspace", []string{}, pb.TabletType_MASTER)
	}
	return &reply, s, nil
}
Exemple #9
0
func (conn *vtgateConn) ExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]interface{}, tabletType pb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
	var s *proto.Session
	if session != nil {
		s = session.(*proto.Session)
	}
	request := proto.QueryShard{
		CallerID:         getEffectiveCallerID(ctx),
		Sql:              query,
		BindVariables:    bindVars,
		Keyspace:         keyspace,
		Shards:           shards,
		TabletType:       topo.ProtoToTabletType(tabletType),
		Session:          s,
		NotInTransaction: notInTransaction,
	}
	var result proto.QueryResult
	if err := conn.rpcConn.Call(ctx, "VTGate.ExecuteShard", request, &result); err != nil {
		return nil, session, err
	}
	if result.Error != "" {
		return nil, result.Session, errors.New(result.Error)
	}
	if err := vterrors.FromRPCError(result.Err); err != nil {
		return nil, result.Session, err
	}
	return result.Result, result.Session, nil
}
Exemple #10
0
func (s *server) RunHealthCheck(ctx context.Context, request *pb.RunHealthCheckRequest) (*pb.RunHealthCheckResponse, error) {
	ctx = callinfo.GRPCCallInfo(ctx)
	response := &pb.RunHealthCheckResponse{}
	return response, s.agent.RPCWrap(ctx, actionnode.TabletActionRunHealthCheck, request, response, func() error {
		s.agent.RunHealthCheck(ctx, topo.ProtoToTabletType(request.TabletType))
		return nil
	})
}
Exemple #11
0
func (conn *vtgateConn) StreamExecute2(ctx context.Context, query string, bindVars map[string]interface{}, tabletType pb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
	req := &proto.Query{
		CallerID:      getEffectiveCallerID(ctx),
		Sql:           query,
		BindVariables: bindVars,
		TabletType:    topo.ProtoToTabletType(tabletType),
		Session:       nil,
	}
	sr := make(chan *proto.QueryResult, 10)
	c := conn.rpcConn.StreamGo("VTGate.StreamExecute2", req, sr)
	return sendStreamResults(c, sr)
}
Exemple #12
0
// Find returns the transactionId, if any, for a session
func (session *SafeSession) Find(keyspace, shard string, tabletType pb.TabletType) int64 {
	if session == nil {
		return 0
	}
	tt := topo.ProtoToTabletType(tabletType)
	session.mu.Lock()
	defer session.mu.Unlock()
	for _, shardSession := range session.ShardSessions {
		if keyspace == shardSession.Keyspace && tt == shardSession.TabletType && shard == shardSession.Shard {
			return shardSession.TransactionId
		}
	}
	return 0
}
Exemple #13
0
// StreamExecute is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGate) StreamExecute(request *pb.StreamExecuteRequest, stream pbs.Vitess_StreamExecuteServer) (err error) {
	defer vtg.server.HandlePanic(&err)

	query := &proto.Query{
		Sql:           string(request.Query.Sql),
		BindVariables: tproto.Proto3ToBindVariables(request.Query.BindVariables),
		TabletType:    topo.ProtoToTabletType(request.TabletType),
	}
	return vtg.server.StreamExecute(stream.Context(), query, func(value *proto.QueryResult) error {
		return stream.Send(&pb.StreamExecuteResponse{
			Result: mproto.QueryResultToProto3(value.Result),
		})
	})
}
Exemple #14
0
// StreamExecute is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGate) StreamExecute(request *pb.StreamExecuteRequest, stream pbs.Vitess_StreamExecuteServer) (err error) {
	defer vtg.server.HandlePanic(&err)
	ctx := callerid.NewContext(callinfo.GRPCCallInfo(stream.Context()),
		request.CallerId,
		callerid.NewImmediateCallerID("grpc client"))
	query := &proto.Query{
		Sql:           string(request.Query.Sql),
		BindVariables: tproto.Proto3ToBindVariables(request.Query.BindVariables),
		TabletType:    topo.ProtoToTabletType(request.TabletType),
	}
	return vtg.server.StreamExecute(ctx, query, func(value *proto.QueryResult) error {
		return stream.Send(&pb.StreamExecuteResponse{
			Result: mproto.QueryResultToProto3(value.Result),
		})
	})
}
Exemple #15
0
// Execute is part of the VTGateService interface
func (f *fakeVTGateService) Execute(ctx context.Context, sql string, bindVariables map[string]interface{}, tabletType pb.TabletType, session *proto.Session, notInTransaction bool, reply *proto.QueryResult) error {
	execCase, ok := execMap[sql]
	if !ok {
		return fmt.Errorf("no match for: %s", sql)
	}
	query := &proto.Query{
		Sql:              sql,
		BindVariables:    bindVariables,
		TabletType:       topo.ProtoToTabletType(tabletType),
		Session:          session,
		NotInTransaction: notInTransaction,
	}
	if !reflect.DeepEqual(query, execCase.execQuery) {
		return fmt.Errorf("request mismatch: got %+v, want %+v", query, execCase.execQuery)
	}
	*reply = *execCase.reply
	return nil
}
Exemple #16
0
// ProtoToSession transforms a proto3 Session into native types
func ProtoToSession(s *pb.Session) *Session {
	if s == nil {
		return nil
	}
	result := &Session{
		InTransaction: s.InTransaction,
	}
	result.ShardSessions = make([]*ShardSession, len(s.ShardSessions))
	for i, ss := range s.ShardSessions {
		result.ShardSessions[i] = &ShardSession{
			Keyspace:      ss.Target.Keyspace,
			Shard:         ss.Target.Shard,
			TabletType:    topo.ProtoToTabletType(ss.Target.TabletType),
			TransactionId: ss.TransactionId,
		}
	}
	return result
}
Exemple #17
0
func newSession(
	inTransaction bool,
	keyspace string,
	shards []string,
	tabletType pb.TabletType) *proto.Session {
	shardSessions := make([]*proto.ShardSession, len(shards))
	for _, shard := range shards {
		shardSessions = append(shardSessions, &proto.ShardSession{
			Keyspace:      keyspace,
			Shard:         shard,
			TabletType:    topo.ProtoToTabletType(tabletType),
			TransactionId: rand.Int63(),
		})
	}
	return &proto.Session{
		InTransaction: inTransaction,
		ShardSessions: shardSessions,
	}
}
Exemple #18
0
func (conn *vtgateConn) ExecuteBatchKeyspaceIds(ctx context.Context, queries []proto.BoundKeyspaceIdQuery, tabletType pb.TabletType, asTransaction bool, session interface{}) ([]mproto.QueryResult, interface{}, error) {
	var s *proto.Session
	if session != nil {
		s = session.(*proto.Session)
	}
	request := proto.KeyspaceIdBatchQuery{
		CallerID:      getEffectiveCallerID(ctx),
		Queries:       queries,
		TabletType:    topo.ProtoToTabletType(tabletType),
		AsTransaction: asTransaction,
		Session:       s,
	}
	var result proto.QueryResultList
	if err := conn.rpcConn.Call(ctx, "VTGate.ExecuteBatchKeyspaceIds", request, &result); err != nil {
		return nil, session, err
	}
	if err := vterrors.FromRPCError(result.Err); err != nil {
		return nil, result.Session, err
	}
	return result.List, result.Session, nil
}
Exemple #19
0
// RefreshTablesByShard calls RefreshState on all the tables of a
// given type in a shard. It would work for the master, but the
// discovery wouldn't be very efficient.
func (wr *Wrangler) RefreshTablesByShard(ctx context.Context, si *topo.ShardInfo, tabletType pb.TabletType, cells []string) error {
	wr.Logger().Infof("RefreshTablesByShard called on shard %v/%v", si.Keyspace(), si.ShardName())
	tabletMap, err := topo.GetTabletMapForShardByCell(ctx, wr.ts, si.Keyspace(), si.ShardName(), cells)
	switch err {
	case nil:
		// keep going
	case topo.ErrPartialResult:
		wr.Logger().Warningf("RefreshTablesByShard: got partial result for shard %v/%v, may not refresh all tablets everywhere", si.Keyspace(), si.ShardName())
	default:
		return err
	}

	// ignore errors in this phase
	wg := sync.WaitGroup{}
	for _, ti := range tabletMap {
		if ti.Type != topo.ProtoToTabletType(tabletType) {
			continue
		}

		wg.Add(1)
		go func(ti *topo.TabletInfo) {
			wr.Logger().Infof("Calling RefreshState on tablet %v", ti.Alias)
			// Setting an upper bound timeout to fail faster in case of an error.
			// Using 60 seconds because RefreshState should not take more than 30 seconds.
			// (RefreshState will restart the tablet's QueryService and most time will be spent on the shutdown, i.e. waiting up to 30 seconds on transactions (see Config.TransactionTimeout)).
			ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
			if err := wr.tmc.RefreshState(ctx, ti); err != nil {
				wr.Logger().Warningf("RefreshTablesByShard: failed to refresh %v: %v", ti.Alias, err)
			}
			cancel()
			wg.Done()
		}(ti)
	}
	wg.Wait()

	return nil
}
Exemple #20
0
// ExecuteBatchShards is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGate) ExecuteBatchShards(ctx context.Context, request *pb.ExecuteBatchShardsRequest) (response *pb.ExecuteBatchShardsResponse, err error) {
	defer vtg.server.HandlePanic(&err)

	query := &proto.BatchQueryShard{
		Session:       proto.ProtoToSession(request.Session),
		Queries:       proto.ProtoToBoundShardQueries(request.Queries),
		TabletType:    topo.ProtoToTabletType(request.TabletType),
		AsTransaction: request.AsTransaction,
	}
	reply := new(proto.QueryResultList)
	executeErr := vtg.server.ExecuteBatchShard(ctx, query, reply)
	response = &pb.ExecuteBatchShardsResponse{
		Error: vtgate.VtGateErrorToVtRPCError(executeErr, reply.Error),
	}
	if executeErr == nil {
		response.Results = tproto.QueryResultListToProto3(reply.List)
		response.Session = proto.SessionToProto(reply.Session)
		return response, nil
	}
	if *vtgate.RPCErrorOnlyInReply {
		return response, nil
	}
	return nil, executeErr
}
Exemple #21
0
// StreamExecute please see vtgateconn.Impl.StreamExecute
func (conn *FakeVTGateConn) StreamExecute(ctx context.Context, query string, bindVars map[string]interface{}, tabletType pb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
	response, ok := conn.execMap[query]
	if !ok {
		return nil, nil, fmt.Errorf("no match for: %s", query)
	}
	queryProto := &proto.Query{
		Sql:           query,
		BindVariables: bindVars,
		TabletType:    topo.ProtoToTabletType(tabletType),
		Session:       nil,
	}
	if !reflect.DeepEqual(queryProto, response.execQuery) {
		return nil, nil, fmt.Errorf("StreamExecute: %+v, want %+v", query, response.execQuery)
	}
	if response.err != nil {
		return nil, nil, response.err
	}
	var resultChan chan *mproto.QueryResult
	defer close(resultChan)
	if response.reply != nil {
		// create a result channel big enough to buffer all of
		// the responses so we don't need to fork a go routine.
		resultChan := make(chan *mproto.QueryResult, len(response.reply.Rows)+1)
		result := &mproto.QueryResult{}
		result.Fields = response.reply.Fields
		resultChan <- result
		for _, row := range response.reply.Rows {
			result := &mproto.QueryResult{}
			result.Rows = [][]sqltypes.Value{row}
			resultChan <- result
		}
	} else {
		resultChan = make(chan *mproto.QueryResult)
	}
	return resultChan, func() error { return nil }, nil
}
Exemple #22
0
func (stc *ScatterConn) updateSession(
	ctx context.Context,
	keyspace, shard string,
	tabletType pb.TabletType,
	session *SafeSession,
	notInTransaction bool,
) (transactionID int64, err error) {
	if !session.InTransaction() {
		return 0, nil
	}
	// No need to protect ourselves from the race condition between
	// Find and Append. The higher level functions ensure that no
	// duplicate (keyspace, shard, tabletType) tuples can execute
	// this at the same time.
	transactionID = session.Find(keyspace, shard, tabletType)
	if transactionID != 0 {
		return transactionID, nil
	}
	// We are in a transaction at higher level,
	// but client requires not to start a transaction for this query.
	// If a transaction was started on this conn, we will use it (as above).
	if notInTransaction {
		return 0, nil
	}
	transactionID, err = stc.gateway.Begin(ctx, keyspace, shard, tabletType)
	if err != nil {
		return 0, err
	}
	session.Append(&proto.ShardSession{
		Keyspace:      keyspace,
		TabletType:    topo.ProtoToTabletType(tabletType),
		Shard:         shard,
		TransactionId: transactionID,
	})
	return transactionID, nil
}
Exemple #23
0
// DbServingGraph returns the ServingGraph for the given cell.
func DbServingGraph(ctx context.Context, ts topo.Server, cell string) (servingGraph *ServingGraph) {
	servingGraph = &ServingGraph{
		Cell:      cell,
		Keyspaces: make(map[string]*KeyspaceNodes),
	}
	rec := concurrency.AllErrorRecorder{}

	keyspaces, err := ts.GetSrvKeyspaceNames(ctx, cell)
	if err != nil {
		servingGraph.Errors = append(servingGraph.Errors, fmt.Sprintf("GetSrvKeyspaceNames failed: %v", err))
		return
	}
	wg := sync.WaitGroup{}
	servingTypes := []topo.TabletType{topo.TYPE_MASTER, topo.TYPE_REPLICA, topo.TYPE_RDONLY}
	for _, keyspace := range keyspaces {
		kn := newKeyspaceNodes()
		servingGraph.Keyspaces[keyspace] = kn
		wg.Add(1)
		go func(keyspace string, kn *KeyspaceNodes) {
			defer wg.Done()

			ks, err := ts.GetSrvKeyspace(ctx, cell, keyspace)
			if err != nil {
				rec.RecordError(fmt.Errorf("GetSrvKeyspace(%v, %v) failed: %v", cell, keyspace, err))
				return
			}
			if len(ks.ServedFrom) > 0 {
				kn.ServedFrom = make(map[topo.TabletType]string)
				for _, sf := range ks.ServedFrom {
					kn.ServedFrom[topo.ProtoToTabletType(sf.TabletType)] = sf.Keyspace
				}
			}

			displayedShards := make(map[string]bool)
			for _, partitionTabletType := range servingTypes {
				kp := topoproto.SrvKeyspaceGetPartition(ks, topo.TabletTypeToProto(partitionTabletType))
				if kp == nil {
					continue
				}
				for _, srvShard := range kp.ShardReferences {
					shard := srvShard.Name
					if displayedShards[shard] {
						continue
					}
					displayedShards[shard] = true

					sn := &ShardNodes{
						Name: shard,
					}
					kn.ShardNodes = append(kn.ShardNodes, sn)
					wg.Add(1)
					go func(shard string, sn *ShardNodes) {
						defer wg.Done()
						tabletTypes, err := ts.GetSrvTabletTypesPerShard(ctx, cell, keyspace, shard)
						if err != nil {
							rec.RecordError(fmt.Errorf("GetSrvTabletTypesPerShard(%v, %v, %v) failed: %v", cell, keyspace, shard, err))
							return
						}
						for _, tabletType := range tabletTypes {
							endPoints, _, err := ts.GetEndPoints(ctx, cell, keyspace, shard, tabletType)
							if err != nil {
								rec.RecordError(fmt.Errorf("GetEndPoints(%v, %v, %v, %v) failed: %v", cell, keyspace, shard, tabletType, err))
								continue
							}
							for _, endPoint := range endPoints.Entries {
								var tabletNode *TabletNodesByType
								for _, t := range sn.TabletNodes {
									if t.TabletType == tabletType {
										tabletNode = t
										break
									}
								}
								if tabletNode == nil {
									tabletNode = &TabletNodesByType{
										TabletType: tabletType,
									}
									sn.TabletNodes = append(sn.TabletNodes, tabletNode)
								}
								tabletNode.Nodes = append(tabletNode.Nodes, newTabletNodeFromEndPoint(endPoint, cell))
							}
						}
					}(shard, sn)
				}
			}
		}(keyspace, kn)
	}
	wg.Wait()
	servingGraph.Errors = rec.ErrorStrings()
	return
}