// makeValueString returns a string that contains all the passed-in rows
// as an insert SQL command's parameters.
func makeValueString(fields []*query.Field, rows [][]sqltypes.Value) string {
buf := bytes.Buffer{}
for i, row := range rows {
if i > 0 {
buf.Write([]byte(",("))
} else {
buf.WriteByte('(')
}
for j, value := range row {
if j > 0 {
buf.WriteByte(',')
}
// convert value back to its original type
if !value.IsNull() {
switch {
case sqltypes.IsIntegral(fields[j].Type):
value = sqltypes.MakeNumeric(value.Raw())
case sqltypes.IsFloat(fields[j].Type):
value = sqltypes.MakeFractional(value.Raw())
}
}
value.EncodeSQL(&buf)
}
buf.WriteByte(')')
}
return buf.String()
}
// NewEqualSplitsAlgorithm constructs a new equal splits algorithm.
// It requires an SQLExecuter since it needs to execute a query to figure out the
// minimum and maximum elements in the table.
func NewEqualSplitsAlgorithm(splitParams *SplitParams, sqlExecuter SQLExecuter) (
*EqualSplitsAlgorithm, error) {
if len(splitParams.splitColumns) != len(splitParams.splitColumnTypes) {
panic(fmt.Sprintf("len(splitparams.splitColumns) != len(splitparams.splitColumnTypes): %v!=%v",
len(splitParams.splitColumns), len(splitParams.splitColumnTypes)))
}
if len(splitParams.splitColumns) != 1 {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires having"+
" exactly one split-column. Got split-columns: %v", splitParams.splitColumns)
}
if !sqltypes.IsFloat(splitParams.splitColumnTypes[0]) &&
!sqltypes.IsIntegral(splitParams.splitColumnTypes[0]) {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires having"+
" a numeric (integral or float) split-column. Got type: %v", splitParams.splitColumnTypes[0])
}
if splitParams.splitCount <= 0 {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires a positive"+
" splitParams.splitCount. Got: %v", splitParams.splitCount)
}
result := &EqualSplitsAlgorithm{
splitParams: splitParams,
sqlExecuter: sqlExecuter,
minMaxQuery: buildMinMaxQuery(splitParams),
}
return result, nil
}
// NewEqualSplitsAlgorithm constructs a new equal splits algorithm.
// It requires an SQLExecuter since it needs to execute a query to figure out the
// minimum and maximum elements in the table.
func NewEqualSplitsAlgorithm(splitParams *SplitParams, sqlExecuter SQLExecuter) (
*EqualSplitsAlgorithm, error) {
if len(splitParams.splitColumns) == 0 {
panic(fmt.Sprintf("len(splitParams.splitColumns) == 0." +
" SplitParams should have defaulted the split columns to the primary key columns."))
}
// This algorithm only uses the first splitColumn.
// Note that we do not force the user to specify only one split column, since a common
// use-case is not to specify split columns at all, which will make them default to the table
// primary key columns, and there can be more than one primary key column for a table.
if !sqltypes.IsFloat(splitParams.splitColumns[0].Type) &&
!sqltypes.IsIntegral(splitParams.splitColumns[0].Type) {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires having"+
" a numeric (integral or float) split-column. Got type: %v", splitParams.splitColumns[0])
}
if splitParams.splitCount <= 0 {
return nil, fmt.Errorf("using the EQUAL_SPLITS algorithm in SplitQuery requires a positive"+
" splitParams.splitCount. Got: %v", splitParams.splitCount)
}
result := &EqualSplitsAlgorithm{
splitParams: splitParams,
sqlExecuter: sqlExecuter,
minMaxQuery: buildMinMaxQuery(splitParams),
}
return result, nil
}
// SQLToNative converts a SQL type & value to a native go type.
// This does not work for sqltypes.Tuple.
func SQLToNative(typ pb.Type, val []byte) (interface{}, error) {
if typ == sqltypes.Null {
return nil, nil
} else if sqltypes.IsSigned(typ) {
return strconv.ParseInt(string(val), 0, 64)
} else if sqltypes.IsUnsigned(typ) {
return strconv.ParseUint(string(val), 0, 64)
} else if sqltypes.IsFloat(typ) {
return strconv.ParseFloat(string(val), 64)
}
return val, nil
}
// Convert takes a type and a value, and returns the type:
// - nil for NULL value
// - uint64 for unsigned BIGINT values
// - int64 for all other integer values (signed and unsigned)
// - float64 for floating point values that fit in a float
// - []byte for everything else
func Convert(field *querypb.Field, val sqltypes.Value) (interface{}, error) {
if field.Type == sqltypes.Null {
return nil, nil
} else if sqltypes.IsSigned(field.Type) {
return strconv.ParseInt(val.String(), 0, 64)
} else if sqltypes.IsUnsigned(field.Type) {
return strconv.ParseUint(val.String(), 0, 64)
} else if sqltypes.IsFloat(field.Type) {
return strconv.ParseFloat(val.String(), 64)
}
return val.Raw(), nil
}
// BindVariableToNative converts a proto bind var to a native go type.
func BindVariableToNative(v *pb.BindVariable) (interface{}, error) {
if v == nil || v.Type == sqltypes.Null {
return nil, nil
} else if sqltypes.IsSigned(v.Type) {
return strconv.ParseInt(string(v.Value), 0, 64)
} else if sqltypes.IsUnsigned(v.Type) {
return strconv.ParseUint(string(v.Value), 0, 64)
} else if sqltypes.IsFloat(v.Type) {
return strconv.ParseFloat(string(v.Value), 64)
}
return v.Value, nil
}
func (qs *QuerySplitter) splitBoundaries(columnType querypb.Type, pkMinMax *sqltypes.Result) ([]sqltypes.Value, error) {
switch {
case sqltypes.IsSigned(columnType):
return qs.splitBoundariesIntColumn(pkMinMax)
case sqltypes.IsUnsigned(columnType):
return qs.splitBoundariesUintColumn(pkMinMax)
case sqltypes.IsFloat(columnType):
return qs.splitBoundariesFloatColumn(pkMinMax)
case sqltypes.IsBinary(columnType) || sqltypes.IsText(columnType):
return qs.splitBoundariesStringColumn()
}
return []sqltypes.Value{}, nil
}
// bigRatToValue converts 'number' to an SQL value with SQL type: valueType.
// If valueType is integral it truncates 'number' to the integer part according to the
// semantics of the big.Rat.Int method.
func bigRatToValue(number *big.Rat, valueType querypb.Type) sqltypes.Value {
var numberAsBytes []byte
switch {
case sqltypes.IsIntegral(valueType):
// 'number.Num()' returns a reference to the numerator of 'number'.
// We copy it here to avoid changing 'number'.
truncatedNumber := new(big.Int).Set(number.Num())
truncatedNumber.Quo(truncatedNumber, number.Denom())
numberAsBytes = bigIntToSliceOfBytes(truncatedNumber)
case sqltypes.IsFloat(valueType):
// Truncate to the closest 'float'.
// There's not much we can do if there isn't an exact representation.
numberAsFloat64, _ := number.Float64()
numberAsBytes = strconv.AppendFloat([]byte{}, numberAsFloat64, 'f', -1, 64)
default:
panic(fmt.Sprintf("Unsupported type: %v", valueType))
}
result, err := sqltypes.ValueFromBytes(valueType, numberAsBytes)
if err != nil {
panic(fmt.Sprintf("sqltypes.ValueFromBytes failed with: %v", err))
}
return result
}
// valueToBigRat converts a numeric 'value' regarded as having type 'valueType' into a
// big.Rat object.
// Note:
// We use an explicit valueType rather than depend on the type stored in 'value' to force
// the type of MAX(column) or MIN(column) to correspond to the type of 'column'.
// (We've had issues where the type of MAX(column) returned by Vitess was signed even if the
// type of column was unsigned).
func valueToBigRat(value sqltypes.Value, valueType querypb.Type) (*big.Rat, error) {
switch {
case sqltypes.IsUnsigned(valueType):
nativeValue, err := value.ParseUint64()
if err != nil {
return nil, err
}
return uint64ToBigRat(nativeValue), nil
case sqltypes.IsSigned(valueType):
nativeValue, err := value.ParseInt64()
if err != nil {
return nil, err
}
return int64ToBigRat(nativeValue), nil
case sqltypes.IsFloat(valueType):
nativeValue, err := value.ParseFloat64()
if err != nil {
return nil, err
}
return float64ToBigRat(nativeValue), nil
default:
panic(fmt.Sprintf("got value with a non numeric type: %v", value))
}
}
// FindChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func FindChunks(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, td *tabletmanagerdatapb.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
result := []string{"", ""}
// eliminate a few cases we don't split tables for
if len(td.PrimaryKeyColumns) == 0 {
// no primary key, what can we do?
return result, nil
}
if td.DataLength < minTableSizeForSplit {
// table is too small to split up
return result, nil
}
// get the min and max of the leading column of the primary key
query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, ti, query, 1, true)
cancel()
if err != nil {
return nil, fmt.Errorf("ExecuteFetchAsApp: %v", err)
}
if len(qr.Rows) != 1 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
return result, nil
}
if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() {
wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1])
return result, nil
}
switch {
case sqltypes.IsSigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
min, err := minNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / int64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := int64(1); i < int64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsUnsigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
min, err := minNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / uint64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := uint64(1); i < uint64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsFloat(qr.Fields[0].Type):
min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err)
return result, nil
}
max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err)
return result, nil
}
interval := (max - min) / float64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := 1; i < sourceReaderCount; i++ {
result[i] = fmt.Sprintf("%v", min+interval*float64(i))
}
return result, nil
}
wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
return result, nil
}
// FindChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func FindChunks(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, td *tabletmanagerdatapb.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
result := []string{"", ""}
// eliminate a few cases we don't split tables for
if len(td.PrimaryKeyColumns) == 0 {
// no primary key, what can we do?
return result, nil
}
if td.DataLength < minTableSizeForSplit {
// table is too small to split up
return result, nil
}
// get the min and max of the leading column of the primary key
query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, ti, query, 1, true)
cancel()
if err != nil {
return nil, fmt.Errorf("ExecuteFetchAsApp: %v", err)
}
if len(qr.Rows) != 1 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
return result, nil
}
// FIXME(alainjobart) this code is a bit clunky. I'd like to
// convert the first row into an array of Values, and then
// see which type they are and go from there. Can only happen after
// Value has a full type.
l0 := qr.Rows[0].Lengths[0]
l1 := qr.Rows[0].Lengths[1]
if l0 < 0 || l1 < 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v", td.Name, qr.Rows[0])
return result, nil
}
minValue := qr.Rows[0].Values[:l0]
maxValue := qr.Rows[0].Values[l0 : l0+l1]
switch {
case sqltypes.IsSigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(minValue)
maxNumeric := sqltypes.MakeNumeric(maxValue)
min, err := minNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / int64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := int64(1); i < int64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsUnsigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(minValue)
maxNumeric := sqltypes.MakeNumeric(maxValue)
min, err := minNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / uint64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := uint64(1); i < uint64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsFloat(qr.Fields[0].Type):
min, err := strconv.ParseFloat(string(minValue), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, string(minValue), err)
return result, nil
}
max, err := strconv.ParseFloat(string(maxValue), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, string(maxValue), err)
return result, nil
}
interval := (max - min) / float64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := 1; i < sourceReaderCount; i++ {
result[i] = fmt.Sprintf("%v", min+interval*float64(i))
}
return result, nil
}
wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
return result, nil
}
