func (ti *TableInfo) initRowCache(conn *DBConn, tableType string, comment string, cachePool *CachePool) {
if cachePool.IsClosed() {
return
}
if strings.Contains(comment, "vitess_nocache") {
log.Infof("%s commented as vitess_nocache. Will not be cached.", ti.Name)
return
}
if tableType == "VIEW" {
log.Infof("%s is a view. Will not be cached.", ti.Name)
return
}
if ti.PKColumns == nil {
log.Infof("Table %s has no primary key. Will not be cached.", ti.Name)
return
}
for _, col := range ti.PKColumns {
if sqltypes.IsIntegral(ti.Columns[col].Type) || ti.Columns[col].Type == sqltypes.VarBinary {
continue
}
log.Infof("Table %s pk has unsupported column types. Will not be cached.", ti.Name)
return
}
ti.CacheType = schema.CacheRW
ti.Cache = NewRowCache(ti, cachePool)
}
// makeValueString returns a string that contains all the passed-in rows
// as an insert SQL command's parameters.
func makeValueString(fields []*query.Field, rows [][]sqltypes.Value) string {
buf := bytes.Buffer{}
for i, row := range rows {
if i > 0 {
buf.Write([]byte(",("))
} else {
buf.WriteByte('(')
}
for j, value := range row {
if j > 0 {
buf.WriteByte(',')
}
// convert value back to its original type
if !value.IsNull() {
switch {
case sqltypes.IsIntegral(fields[j].Type):
value = sqltypes.MakeNumeric(value.Raw())
case sqltypes.IsFloat(fields[j].Type):
value = sqltypes.MakeFractional(value.Raw())
}
}
value.EncodeSQL(&buf)
}
buf.WriteByte(')')
}
return buf.String()
}
// NewEqualSplitsAlgorithm constructs a new equal splits algorithm.
// It requires an SQLExecuter since it needs to execute a query to figure out the
// minimum and maximum elements in the table.
func NewEqualSplitsAlgorithm(splitParams *SplitParams, sqlExecuter SQLExecuter) (
*EqualSplitsAlgorithm, error) {
if len(splitParams.splitColumns) != len(splitParams.splitColumnTypes) {
panic(fmt.Sprintf("len(splitparams.splitColumns) != len(splitparams.splitColumnTypes): %v!=%v",
len(splitParams.splitColumns), len(splitParams.splitColumnTypes)))
}
if len(splitParams.splitColumns) != 1 {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires having"+
" exactly one split-column. Got split-columns: %v", splitParams.splitColumns)
}
if !sqltypes.IsFloat(splitParams.splitColumnTypes[0]) &&
!sqltypes.IsIntegral(splitParams.splitColumnTypes[0]) {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires having"+
" a numeric (integral or float) split-column. Got type: %v", splitParams.splitColumnTypes[0])
}
if splitParams.splitCount <= 0 {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires a positive"+
" splitParams.splitCount. Got: %v", splitParams.splitCount)
}
result := &EqualSplitsAlgorithm{
splitParams: splitParams,
sqlExecuter: sqlExecuter,
minMaxQuery: buildMinMaxQuery(splitParams),
}
return result, nil
}
// NewEqualSplitsAlgorithm constructs a new equal splits algorithm.
// It requires an SQLExecuter since it needs to execute a query to figure out the
// minimum and maximum elements in the table.
func NewEqualSplitsAlgorithm(splitParams *SplitParams, sqlExecuter SQLExecuter) (
*EqualSplitsAlgorithm, error) {
if len(splitParams.splitColumns) == 0 {
panic(fmt.Sprintf("len(splitParams.splitColumns) == 0." +
" SplitParams should have defaulted the split columns to the primary key columns."))
}
// This algorithm only uses the first splitColumn.
// Note that we do not force the user to specify only one split column, since a common
// use-case is not to specify split columns at all, which will make them default to the table
// primary key columns, and there can be more than one primary key column for a table.
if !sqltypes.IsFloat(splitParams.splitColumns[0].Type) &&
!sqltypes.IsIntegral(splitParams.splitColumns[0].Type) {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires having"+
" a numeric (integral or float) split-column. Got type: %v", splitParams.splitColumns[0])
}
if splitParams.splitCount <= 0 {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires a positive"+
" splitParams.splitCount. Got: %v", splitParams.splitCount)
}
result := &EqualSplitsAlgorithm{
splitParams: splitParams,
sqlExecuter: sqlExecuter,
minMaxQuery: buildMinMaxQuery(splitParams),
}
return result, nil
}
func validateValue(col *schema.TableColumn, value sqltypes.Value) error {
if value.IsNull() {
return nil
}
if sqltypes.IsIntegral(col.Type) {
if !value.IsNumeric() {
return NewTabletError(ErrFail, vtrpc.ErrorCode_BAD_INPUT, "type mismatch, expecting numeric type for %v for column: %v", value, col)
}
} else if col.Type == sqltypes.VarBinary {
if !value.IsString() {
return NewTabletError(ErrFail, vtrpc.ErrorCode_BAD_INPUT, "type mismatch, expecting string type for %v for column: %v", value, col)
}
}
return nil
}
// SplitQuery splits a query + bind variables into smaller queries that return a
// subset of rows from the original query.
// TODO(erez): Remove this method and rename SplitQueryV2 to SplitQuery once we migrate to
// SplitQuery V2.
func (tsv *TabletServer) SplitQuery(ctx context.Context, target *querypb.Target, sql string, bindVariables map[string]interface{}, splitColumn string, splitCount int64) (splits []querytypes.QuerySplit, err error) {
logStats := newLogStats("SplitQuery", ctx)
logStats.OriginalSQL = sql
logStats.BindVariables = bindVariables
defer handleError(&err, logStats, tsv.qe.queryServiceStats)
if err = tsv.startRequest(target, false, false); err != nil {
return nil, err
}
ctx, cancel := withTimeout(ctx, tsv.QueryTimeout.Get())
defer func() {
cancel()
tsv.endRequest(false)
}()
splitter := NewQuerySplitter(sql, bindVariables, splitColumn, splitCount, tsv.qe.schemaInfo)
err = splitter.validateQuery()
if err != nil {
return nil, NewTabletError(vtrpcpb.ErrorCode_BAD_INPUT, "splitQuery: query validation error: %s, request: %v", err, querytypes.QueryAsString(sql, bindVariables))
}
defer func(start time.Time) {
addUserTableQueryStats(tsv.qe.queryServiceStats, ctx, splitter.tableName, "SplitQuery", int64(time.Now().Sub(start)))
}(time.Now())
qre := &QueryExecutor{
ctx: ctx,
logStats: logStats,
qe: tsv.qe,
}
columnType, err := getColumnType(qre, splitter.splitColumn, splitter.tableName)
if err != nil {
return nil, err
}
var pkMinMax *sqltypes.Result
if sqltypes.IsIntegral(columnType) {
pkMinMax, err = getColumnMinMax(qre, splitter.splitColumn, splitter.tableName)
if err != nil {
return nil, err
}
}
splits, err = splitter.split(columnType, pkMinMax)
if err != nil {
return nil, NewTabletError(vtrpcpb.ErrorCode_BAD_INPUT, "splitQuery: query split error: %s, request: %v", err, querytypes.QueryAsString(sql, bindVariables))
}
return splits, nil
}
func createTableInfo(
name string, colNames []string, colTypes []querypb.Type, pKeys []string) TableInfo {
table := schema.NewTable(name)
for i, colName := range colNames {
colType := colTypes[i]
defaultVal := sqltypes.Value{}
if sqltypes.IsIntegral(colType) {
defaultVal = sqltypes.MakeTrusted(sqltypes.Int64, []byte("0"))
} else if colType == sqltypes.VarBinary {
defaultVal = sqltypes.MakeString([]byte(""))
}
table.AddColumn(colName, colType, defaultVal, "")
}
tableInfo := TableInfo{Table: table}
tableInfo.SetPK(pKeys)
return tableInfo
}
// AddColumn adds a column to the Table.
func (ta *Table) AddColumn(name string, columnType querypb.Type, defval sqltypes.Value, extra string) {
index := len(ta.Columns)
ta.Columns = append(ta.Columns, TableColumn{Name: name})
ta.Columns[index].Type = columnType
if extra == "auto_increment" {
ta.Columns[index].IsAuto = true
// Ignore default value, if any
return
}
if defval.IsNull() {
return
}
if sqltypes.IsIntegral(ta.Columns[index].Type) {
ta.Columns[index].Default = sqltypes.MakeNumeric(defval.Raw())
} else {
ta.Columns[index].Default = sqltypes.MakeString(defval.Raw())
}
}
// SplitQuery splits a BoundQuery into smaller queries that return a subset of rows from the original query.
func (tsv *TabletServer) SplitQuery(ctx context.Context, target *pbq.Target, req *proto.SplitQueryRequest, reply *proto.SplitQueryResult) (err error) {
logStats := newLogStats("SplitQuery", ctx)
defer handleError(&err, logStats, tsv.qe.queryServiceStats)
if err = tsv.startRequest(target, req.SessionID, false, false); err != nil {
return err
}
ctx, cancel := withTimeout(ctx, tsv.QueryTimeout.Get())
defer func() {
cancel()
tsv.endRequest(false)
}()
splitter := NewQuerySplitter(&(req.Query), req.SplitColumn, req.SplitCount, tsv.qe.schemaInfo)
err = splitter.validateQuery()
if err != nil {
return NewTabletError(ErrFail, vtrpc.ErrorCode_BAD_INPUT, "splitQuery: query validation error: %s, request: %#v", err, req)
}
defer func(start time.Time) {
addUserTableQueryStats(tsv.qe.queryServiceStats, ctx, splitter.tableName, "SplitQuery", int64(time.Now().Sub(start)))
}(time.Now())
qre := &QueryExecutor{
ctx: ctx,
logStats: logStats,
qe: tsv.qe,
}
columnType, err := getColumnType(qre, splitter.splitColumn, splitter.tableName)
if err != nil {
return err
}
var pkMinMax *sqltypes.Result
if sqltypes.IsIntegral(columnType) {
pkMinMax, err = getColumnMinMax(qre, splitter.splitColumn, splitter.tableName)
if err != nil {
return err
}
}
reply.Queries, err = splitter.split(columnType, pkMinMax)
if err != nil {
return NewTabletError(ErrFail, vtrpc.ErrorCode_BAD_INPUT, "splitQuery: query split error: %s, request: %#v", err, req)
}
return nil
}
func (rc *RowCache) decodeRow(b []byte) (row []sqltypes.Value) {
rowlen := pack.Uint32(b)
data := b[4+rowlen*4:]
row = make([]sqltypes.Value, rowlen)
for i := range row {
length := pack.Uint32(b[4+i*4:])
if length == 0xFFFFFFFF {
continue
}
if length > uint32(len(data)) {
// Corrupt data
return nil
}
if sqltypes.IsIntegral(rc.tableInfo.Columns[i].Type) {
row[i] = sqltypes.MakeNumeric(data[:length])
} else {
row[i] = sqltypes.MakeString(data[:length])
}
data = data[length:]
}
return row
}
// bigRatToValue converts 'number' to an SQL value with SQL type: valueType.
// If valueType is integral it truncates 'number' to the integer part according to the
// semantics of the big.Rat.Int method.
func bigRatToValue(number *big.Rat, valueType querypb.Type) sqltypes.Value {
var numberAsBytes []byte
switch {
case sqltypes.IsIntegral(valueType):
// 'number.Num()' returns a reference to the numerator of 'number'.
// We copy it here to avoid changing 'number'.
truncatedNumber := new(big.Int).Set(number.Num())
truncatedNumber.Quo(truncatedNumber, number.Denom())
numberAsBytes = bigIntToSliceOfBytes(truncatedNumber)
case sqltypes.IsFloat(valueType):
// Truncate to the closest 'float'.
// There's not much we can do if there isn't an exact representation.
numberAsFloat64, _ := number.Float64()
numberAsBytes = strconv.AppendFloat([]byte{}, numberAsFloat64, 'f', -1, 64)
default:
panic(fmt.Sprintf("Unsupported type: %v", valueType))
}
result, err := sqltypes.ValueFromBytes(valueType, numberAsBytes)
if err != nil {
panic(fmt.Sprintf("sqltypes.ValueFromBytes failed with: %v", err))
}
return result
}
func (a *EqualSplitsAlgorithm) generateBoundaries() ([]tuple, error) {
// generateBoundaries should work for a split_column whose type is integral
// (both signed and unsigned) as well as for floating point values.
// We perform the calculation of the boundaries using precise big.Rat arithmetic and only
// truncate the result in the end if necessary.
// We do this since using float64 arithmetic does not have enough precision:
// for example, if max=math.MaxUint64 and min=math.MaxUint64-1000 then float64(min)==float64(max).
// On the other hand, using integer arithmetic for the case where the split_column is integral
// (i.e., rounding (max-min)/split_count to an integer) may cause very dissimilar interval
// lengths or a large deviation between split_count and the number of query-parts actually
// returned (consider min=0, max=9.5*10^6, and split_count=10^6).
// Note(erez): We can probably get away with using big.Float with ~64 bits of precision which
// will likely be more efficient. However, we defer optimizing this code until we see if this
// is a bottle-neck.
minValue, maxValue, err := a.executeMinMaxQuery()
if err != nil {
return nil, err
}
// If the table is empty, minValue and maxValue will be NULL.
if (minValue.IsNull() && !maxValue.IsNull()) ||
!minValue.IsNull() && maxValue.IsNull() {
panic(fmt.Sprintf("minValue and maxValue must both be NULL or both be non-NULL."+
" minValue: %v, maxValue: %v, splitParams.sql: %v",
minValue, maxValue, a.splitParams.sql))
}
if minValue.IsNull() {
log.Infof("Splitting an empty table. splitParams.sql: %v. Query will not be split.",
a.splitParams.sql)
return []tuple{}, nil
}
min, err := valueToBigRat(minValue, a.splitParams.splitColumns[0].Type)
if err != nil {
panic(fmt.Sprintf("Failed to convert min to a big.Rat: %v, min: %+v", err, min))
}
max, err := valueToBigRat(maxValue, a.splitParams.splitColumns[0].Type)
if err != nil {
panic(fmt.Sprintf("Failed to convert max to a big.Rat: %v, max: %+v", err, max))
}
minCmpMax := min.Cmp(max)
if minCmpMax > 0 {
panic(fmt.Sprintf("max(splitColumn) < min(splitColumn): max:%v, min:%v", max, min))
}
if minCmpMax == 0 {
log.Infof("max(%v)=min(%v)=%v. splitParams.sql: %v. Query will not be split.",
a.splitParams.splitColumns[0].Name,
a.splitParams.splitColumns[0].Name,
min,
a.splitParams.sql)
return []tuple{}, nil
}
// subIntervalSize = (max - min) / splitCount
maxMinDiff := new(big.Rat)
maxMinDiff.Sub(max, min)
subIntervalSize := new(big.Rat)
subIntervalSize.Quo(maxMinDiff, new(big.Rat).SetInt64(a.splitParams.splitCount))
// If the split-column type is integral then it's wasteful to have a sub-intervale-size smaller
// than 1, as it'll result with some query-parts being trivially empty. We set the
// sub-interval size to 1 in this case.
one := new(big.Rat).SetInt64(1)
if sqltypes.IsIntegral(a.splitParams.splitColumns[0].Type) &&
subIntervalSize.Cmp(one) < 0 {
subIntervalSize = one
}
boundary := new(big.Rat).Add(min, subIntervalSize)
result := []tuple{}
for ; boundary.Cmp(max) < 0; boundary.Add(boundary, subIntervalSize) {
boundaryValue := bigRatToValue(boundary, a.splitParams.splitColumns[0].Type)
result = append(result, tuple{boundaryValue})
}
return result, nil
}