// getQuerySplitToKeyRangePartFunc returns a function to use with scatterConn.SplitQueryV2
// that converts the given QuerySplit to a SplitQueryResponse_Part message whose KeyRangePart field
// is set.
func getQuerySplitToKeyRangePartFunc(
keyspace string,
shardReferenceByName map[string]*topodatapb.ShardReference) func(
querySplit *querytypes.QuerySplit, shard string) (*vtgatepb.SplitQueryResponse_Part, error) {
return func(
querySplit *querytypes.QuerySplit, shard string) (*vtgatepb.SplitQueryResponse_Part, error) {
// TODO(erez): Assert that shardReferenceByName contains an entry for 'shard'.
// Keyrange can be nil for the shard (e.g. for single-sharded keyspaces during resharding).
// In this case we append an empty keyrange that represents the entire keyspace.
keyranges := []*topodatapb.KeyRange{{Start: []byte{}, End: []byte{}}}
if shardReferenceByName[shard].KeyRange != nil {
keyranges = []*topodatapb.KeyRange{shardReferenceByName[shard].KeyRange}
}
bindVars, err := querytypes.BindVariablesToProto3(querySplit.BindVariables)
if err != nil {
return nil, err
}
return &vtgatepb.SplitQueryResponse_Part{
Query: &querypb.BoundQuery{
Sql: querySplit.Sql,
BindVariables: bindVars,
},
KeyRangePart: &vtgatepb.SplitQueryResponse_KeyRangePart{
Keyspace: keyspace,
KeyRanges: keyranges,
},
Size: querySplit.RowCount,
}, nil
}
}
// StreamExecute is part of tabletconn.TabletConn
// We need to copy the bind variables as tablet server will change them.
func (itc *internalTabletConn) StreamExecute(ctx context.Context, query string, bindVars map[string]interface{}, transactionID int64) (<-chan *sqltypes.Result, tabletconn.ErrFunc, error) {
bv, err := querytypes.BindVariablesToProto3(bindVars)
if err != nil {
return nil, nil, err
}
bindVars, err = querytypes.Proto3ToBindVariables(bv)
if err != nil {
return nil, nil, err
}
result := make(chan *sqltypes.Result, 10)
var finalErr error
go func() {
finalErr = itc.tablet.qsc.QueryService().StreamExecute(ctx, &querypb.Target{
Keyspace: itc.tablet.keyspace,
Shard: itc.tablet.shard,
TabletType: itc.tablet.tabletType,
}, query, bindVars, 0, func(reply *sqltypes.Result) error {
// We need to deep-copy the reply before returning,
// because the underlying buffers are reused.
result <- reply.Copy()
return nil
})
// the client will only access finalErr after the
// channel is closed, and then it's already set.
close(result)
}()
return result, func() error {
return tabletconn.TabletErrorFromGRPC(tabletserver.ToGRPCError(finalErr))
}, nil
}
// StreamExecute is part of tabletconn.TabletConn
// We need to copy the bind variables as tablet server will change them.
func (itc *internalTabletConn) StreamExecute(ctx context.Context, target *querypb.Target, query string, bindVars map[string]interface{}, options *querypb.ExecuteOptions) (sqltypes.ResultStream, error) {
bv, err := querytypes.BindVariablesToProto3(bindVars)
if err != nil {
return nil, err
}
bindVars, err = querytypes.Proto3ToBindVariables(bv)
if err != nil {
return nil, err
}
result := make(chan *sqltypes.Result, 10)
var finalErr error
go func() {
finalErr = itc.tablet.qsc.QueryService().StreamExecute(ctx, target, query, bindVars, options, func(reply *sqltypes.Result) error {
// We need to deep-copy the reply before returning,
// because the underlying buffers are reused.
result <- reply.Copy()
return nil
})
finalErr = tabletconn.TabletErrorFromGRPC(vterrors.ToGRPCError(finalErr))
// the client will only access finalErr after the
// channel is closed, and then it's already set.
close(result)
}()
return &streamExecuteAdapter{result, &finalErr}, nil
}
// SplitQueryKeyRange scatters a SplitQuery request to all shards. For a set of
// splits received from a shard, it construct a KeyRange queries by
// appending that shard's keyrange to the splits. Aggregates all splits across
// all shards in no specific order and returns.
func (stc *ScatterConn) SplitQueryKeyRange(ctx context.Context, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int64, keyRangeByShard map[string]*topodatapb.KeyRange, keyspace string) ([]*vtgatepb.SplitQueryResponse_Part, error) {
tabletType := topodatapb.TabletType_RDONLY
actionFunc := func(shard string, transactionID int64, results chan<- interface{}) error {
// Get all splits from this shard
queries, err := stc.gateway.SplitQuery(ctx, keyspace, shard, tabletType, sql, bindVariables, splitColumn, splitCount)
if err != nil {
return err
}
// Append the keyrange for this shard to all the splits received,
// if keyrange is nil for the shard (e.g. for single-sharded keyspaces during resharding),
// append empty keyrange to represent the entire keyspace.
keyranges := []*topodatapb.KeyRange{{Start: []byte{}, End: []byte{}}}
if keyRangeByShard[shard] != nil {
keyranges = []*topodatapb.KeyRange{keyRangeByShard[shard]}
}
splits := make([]*vtgatepb.SplitQueryResponse_Part, len(queries))
for i, query := range queries {
q, err := querytypes.BindVariablesToProto3(query.BindVariables)
if err != nil {
return err
}
splits[i] = &vtgatepb.SplitQueryResponse_Part{
Query: &querypb.BoundQuery{
Sql: query.Sql,
BindVariables: q,
},
KeyRangePart: &vtgatepb.SplitQueryResponse_KeyRangePart{
Keyspace: keyspace,
KeyRanges: keyranges,
},
Size: query.RowCount,
}
}
// Push all the splits from this shard to results channel
results <- splits
return nil
}
shards := []string{}
for shard := range keyRangeByShard {
shards = append(shards, shard)
}
allSplits, allErrors := stc.multiGo(ctx, "SplitQuery", keyspace, shards, tabletType, NewSafeSession(&vtgatepb.Session{}), false, actionFunc)
splits := []*vtgatepb.SplitQueryResponse_Part{}
for s := range allSplits {
splits = append(splits, s.([]*vtgatepb.SplitQueryResponse_Part)...)
}
if allErrors.HasErrors() {
err := allErrors.AggrError(stc.aggregateErrors)
return nil, err
}
return splits, nil
}
// SplitQueryCustomSharding scatters a SplitQuery request to all
// shards. For a set of splits received from a shard, it construct a
// KeyRange queries by appending that shard's name to the
// splits. Aggregates all splits across all shards in no specific
// order and returns.
func (stc *ScatterConn) SplitQueryCustomSharding(ctx context.Context, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int64, shards []string, keyspace string) ([]*vtgatepb.SplitQueryResponse_Part, error) {
tabletType := topodatapb.TabletType_RDONLY
// mu protects allSplits
var mu sync.Mutex
var allSplits []*vtgatepb.SplitQueryResponse_Part
actionFunc := func(target *querypb.Target) error {
// Get all splits from this shard
query := querytypes.BoundQuery{
Sql: sql,
BindVariables: bindVariables,
}
queries, err := stc.gateway.SplitQuery(ctx, target, query, splitColumn, splitCount)
if err != nil {
return err
}
// Use the shards list for all the splits received
shards := []string{target.Shard}
splits := make([]*vtgatepb.SplitQueryResponse_Part, len(queries))
for i, query := range queries {
q, err := querytypes.BindVariablesToProto3(query.BindVariables)
if err != nil {
return err
}
splits[i] = &vtgatepb.SplitQueryResponse_Part{
Query: &querypb.BoundQuery{
Sql: query.Sql,
BindVariables: q,
},
ShardPart: &vtgatepb.SplitQueryResponse_ShardPart{
Keyspace: keyspace,
Shards: shards,
},
Size: query.RowCount,
}
}
// aggregate splits
mu.Lock()
defer mu.Unlock()
allSplits = append(allSplits, splits...)
return nil
}
allErrors := stc.multiGo(ctx, "SplitQuery", keyspace, shards, tabletType, actionFunc)
if allErrors.HasErrors() {
return nil, allErrors.AggrError(stc.aggregateErrors)
}
// See the comment for the analogues line in SplitQueryKeyRange for
// the motivation for shuffling.
shuffleQueryParts(allSplits)
return allSplits, nil
}
// Execute is part of tabletconn.TabletConn
// We need to copy the bind variables as tablet server will change them.
func (itc *internalTabletConn) Execute(ctx context.Context, target *querypb.Target, query string, bindVars map[string]interface{}, transactionID int64, options *querypb.ExecuteOptions) (*sqltypes.Result, error) {
bv, err := querytypes.BindVariablesToProto3(bindVars)
if err != nil {
return nil, err
}
bindVars, err = querytypes.Proto3ToBindVariables(bv)
if err != nil {
return nil, err
}
reply, err := itc.tablet.qsc.QueryService().Execute(ctx, target, query, bindVars, transactionID, options)
if err != nil {
return nil, tabletconn.TabletErrorFromGRPC(vterrors.ToGRPCError(err))
}
return reply, nil
}
// SplitQueryCustomSharding scatters a SplitQuery request to all
// shards. For a set of splits received from a shard, it construct a
// KeyRange queries by appending that shard's name to the
// splits. Aggregates all splits across all shards in no specific
// order and returns.
func (stc *ScatterConn) SplitQueryCustomSharding(ctx context.Context, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int64, shards []string, keyspace string) ([]*vtgatepb.SplitQueryResponse_Part, error) {
tabletType := topodatapb.TabletType_RDONLY
actionFunc := func(shard string, transactionID int64, results chan<- interface{}) error {
// Get all splits from this shard
queries, err := stc.gateway.SplitQuery(ctx, keyspace, shard, tabletType, sql, bindVariables, splitColumn, splitCount)
if err != nil {
return err
}
// Use the shards list for all the splits received
shards := []string{shard}
splits := make([]*vtgatepb.SplitQueryResponse_Part, len(queries))
for i, query := range queries {
q, err := querytypes.BindVariablesToProto3(query.BindVariables)
if err != nil {
return err
}
splits[i] = &vtgatepb.SplitQueryResponse_Part{
Query: &querypb.BoundQuery{
Sql: query.Sql,
BindVariables: q,
},
ShardPart: &vtgatepb.SplitQueryResponse_ShardPart{
Keyspace: keyspace,
Shards: shards,
},
Size: query.RowCount,
}
}
// Push all the splits from this shard to results channel
results <- splits
return nil
}
allSplits, allErrors := stc.multiGo(ctx, "SplitQuery", keyspace, shards, tabletType, NewSafeSession(&vtgatepb.Session{}), false, actionFunc)
splits := []*vtgatepb.SplitQueryResponse_Part{}
for s := range allSplits {
splits = append(splits, s.([]*vtgatepb.SplitQueryResponse_Part)...)
}
if allErrors.HasErrors() {
err := allErrors.AggrError(stc.aggregateErrors)
return nil, err
}
return splits, nil
}
// Execute is part of tabletconn.TabletConn
// We need to copy the bind variables as tablet server will change them.
func (itc *internalTabletConn) Execute(ctx context.Context, query string, bindVars map[string]interface{}, transactionID int64) (*sqltypes.Result, error) {
bv, err := querytypes.BindVariablesToProto3(bindVars)
if err != nil {
return nil, err
}
bindVars, err = querytypes.Proto3ToBindVariables(bv)
if err != nil {
return nil, err
}
reply, err := itc.tablet.qsc.QueryService().Execute(ctx, &querypb.Target{
Keyspace: itc.tablet.keyspace,
Shard: itc.tablet.shard,
TabletType: itc.tablet.tabletType,
}, query, bindVars, 0, transactionID)
if err != nil {
return nil, tabletconn.TabletErrorFromGRPC(tabletserver.ToGRPCError(err))
}
return reply, nil
}
func (c *echoClient) SplitQuery(ctx context.Context, keyspace string, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int64) ([]*vtgatepb.SplitQueryResponse_Part, error) {
if strings.HasPrefix(sql, EchoPrefix) {
bv, err := querytypes.BindVariablesToProto3(bindVariables)
if err != nil {
return nil, err
}
return []*vtgatepb.SplitQueryResponse_Part{
{
Query: &querypb.BoundQuery{
Sql: fmt.Sprintf("%v:%v:%v", sql, splitColumn, splitCount),
BindVariables: bv,
},
KeyRangePart: &vtgatepb.SplitQueryResponse_KeyRangePart{
Keyspace: keyspace,
},
},
}, nil
}
return c.fallback.SplitQuery(ctx, sql, keyspace, bindVariables, splitColumn, splitCount)
}
// ExecuteBatch is part of tabletconn.TabletConn
// We need to copy the bind variables as tablet server will change them.
func (itc *internalTabletConn) ExecuteBatch(ctx context.Context, target *querypb.Target, queries []querytypes.BoundQuery, asTransaction bool, transactionID int64, options *querypb.ExecuteOptions) ([]sqltypes.Result, error) {
q := make([]querytypes.BoundQuery, len(queries))
for i, query := range queries {
bv, err := querytypes.BindVariablesToProto3(query.BindVariables)
if err != nil {
return nil, err
}
bindVars, err := querytypes.Proto3ToBindVariables(bv)
if err != nil {
return nil, err
}
q[i].Sql = query.Sql
q[i].BindVariables = bindVars
}
results, err := itc.tablet.qsc.QueryService().ExecuteBatch(ctx, target, q, asTransaction, transactionID, options)
if err != nil {
return nil, tabletconn.TabletErrorFromGRPC(vterrors.ToGRPCError(err))
}
return results, nil
}
// getQuerySplitToShardPartFunc returns a function to use with scatterConn.SplitQueryV2
// that converts the given QuerySplit to a SplitQueryResponse_Part message whose ShardPart field
// is set.
func getQuerySplitToShardPartFunc(keyspace string) func(
querySplit *querytypes.QuerySplit, shard string) (*vtgatepb.SplitQueryResponse_Part, error) {
return func(
querySplit *querytypes.QuerySplit, shard string) (*vtgatepb.SplitQueryResponse_Part, error) {
bindVars, err := querytypes.BindVariablesToProto3(querySplit.BindVariables)
if err != nil {
return nil, err
}
return &vtgatepb.SplitQueryResponse_Part{
Query: &querypb.BoundQuery{
Sql: querySplit.Sql,
BindVariables: bindVars,
},
ShardPart: &vtgatepb.SplitQueryResponse_ShardPart{
Keyspace: keyspace,
Shards: []string{shard},
},
Size: querySplit.RowCount,
}, nil
}
}
// ExecuteBatch is part of tabletconn.TabletConn
// We need to copy the bind variables as tablet server will change them.
func (itc *internalTabletConn) ExecuteBatch(ctx context.Context, queries []querytypes.BoundQuery, asTransaction bool, transactionID int64) ([]sqltypes.Result, error) {
q := make([]querytypes.BoundQuery, len(queries))
for i, query := range queries {
bv, err := querytypes.BindVariablesToProto3(query.BindVariables)
if err != nil {
return nil, err
}
bindVars, err := querytypes.Proto3ToBindVariables(bv)
if err != nil {
return nil, err
}
q[i].Sql = query.Sql
q[i].BindVariables = bindVars
}
results, err := itc.tablet.qsc.QueryService().ExecuteBatch(ctx, &querypb.Target{
Keyspace: itc.tablet.keyspace,
Shard: itc.tablet.shard,
TabletType: itc.tablet.tabletType,
}, q, 0, asTransaction, transactionID)
if err != nil {
return nil, tabletconn.TabletErrorFromGRPC(tabletserver.ToGRPCError(err))
}
return results, nil
}
// SplitQueryKeyRange scatters a SplitQuery request to all shards. For a set of
// splits received from a shard, it construct a KeyRange queries by
// appending that shard's keyrange to the splits. Aggregates all splits across
// all shards in no specific order and returns.
func (stc *ScatterConn) SplitQueryKeyRange(ctx context.Context, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int64, keyRangeByShard map[string]*topodatapb.KeyRange, keyspace string) ([]*vtgatepb.SplitQueryResponse_Part, error) {
tabletType := topodatapb.TabletType_RDONLY
// mu protects allSplits
var mu sync.Mutex
var allSplits []*vtgatepb.SplitQueryResponse_Part
actionFunc := func(shard string, transactionID int64) error {
// Get all splits from this shard
queries, err := stc.gateway.SplitQuery(ctx, keyspace, shard, tabletType, sql, bindVariables, splitColumn, splitCount)
if err != nil {
return err
}
// Append the keyrange for this shard to all the splits received,
// if keyrange is nil for the shard (e.g. for single-sharded keyspaces during resharding),
// append empty keyrange to represent the entire keyspace.
keyranges := []*topodatapb.KeyRange{{Start: []byte{}, End: []byte{}}}
if keyRangeByShard[shard] != nil {
keyranges = []*topodatapb.KeyRange{keyRangeByShard[shard]}
}
splits := make([]*vtgatepb.SplitQueryResponse_Part, len(queries))
for i, query := range queries {
q, err := querytypes.BindVariablesToProto3(query.BindVariables)
if err != nil {
return err
}
splits[i] = &vtgatepb.SplitQueryResponse_Part{
Query: &querypb.BoundQuery{
Sql: query.Sql,
BindVariables: q,
},
KeyRangePart: &vtgatepb.SplitQueryResponse_KeyRangePart{
Keyspace: keyspace,
KeyRanges: keyranges,
},
Size: query.RowCount,
}
}
// aggregate splits
mu.Lock()
defer mu.Unlock()
allSplits = append(allSplits, splits...)
return nil
}
shards := []string{}
for shard := range keyRangeByShard {
shards = append(shards, shard)
}
allErrors := stc.multiGo(ctx, "SplitQuery", keyspace, shards, tabletType, NewSafeSession(&vtgatepb.Session{}), false, actionFunc)
if allErrors.HasErrors() {
return nil, allErrors.AggrError(stc.aggregateErrors)
}
// We shuffle the query-parts here. External frameworks like MapReduce may
// "deal" these jobs to workers in the order they are in the list. Without
// shuffling workers can be very unevenly distributed among
// the shards they query. E.g. all workers will first query the first shard,
// then most of them to the second shard, etc, which results with uneven
// load balancing among shards.
shuffleQueryParts(allSplits)
return allSplits, nil
}