// SQLToNative converts a SQL type & value to a native go type.
// This does not work for sqltypes.Tuple.
func SQLToNative(typ pb.Type, val []byte) (interface{}, error) {
if typ == sqltypes.Null {
return nil, nil
} else if sqltypes.IsSigned(typ) {
return strconv.ParseInt(string(val), 0, 64)
} else if sqltypes.IsUnsigned(typ) {
return strconv.ParseUint(string(val), 0, 64)
} else if sqltypes.IsFloat(typ) {
return strconv.ParseFloat(string(val), 64)
}
return val, nil
}
// Convert takes a type and a value, and returns the type:
// - nil for NULL value
// - uint64 for unsigned BIGINT values
// - int64 for all other integer values (signed and unsigned)
// - float64 for floating point values that fit in a float
// - []byte for everything else
func Convert(field *querypb.Field, val sqltypes.Value) (interface{}, error) {
if field.Type == sqltypes.Null {
return nil, nil
} else if sqltypes.IsSigned(field.Type) {
return strconv.ParseInt(val.String(), 0, 64)
} else if sqltypes.IsUnsigned(field.Type) {
return strconv.ParseUint(val.String(), 0, 64)
} else if sqltypes.IsFloat(field.Type) {
return strconv.ParseFloat(val.String(), 64)
}
return val.Raw(), nil
}
// BindVariableToNative converts a proto bind var to a native go type.
func BindVariableToNative(v *pb.BindVariable) (interface{}, error) {
if v == nil || v.Type == sqltypes.Null {
return nil, nil
} else if sqltypes.IsSigned(v.Type) {
return strconv.ParseInt(string(v.Value), 0, 64)
} else if sqltypes.IsUnsigned(v.Type) {
return strconv.ParseUint(string(v.Value), 0, 64)
} else if sqltypes.IsFloat(v.Type) {
return strconv.ParseFloat(string(v.Value), 64)
}
return v.Value, nil
}
func (qs *QuerySplitter) splitBoundaries(columnType querypb.Type, pkMinMax *sqltypes.Result) ([]sqltypes.Value, error) {
switch {
case sqltypes.IsSigned(columnType):
return qs.splitBoundariesIntColumn(pkMinMax)
case sqltypes.IsUnsigned(columnType):
return qs.splitBoundariesUintColumn(pkMinMax)
case sqltypes.IsFloat(columnType):
return qs.splitBoundariesFloatColumn(pkMinMax)
case sqltypes.IsBinary(columnType) || sqltypes.IsText(columnType):
return qs.splitBoundariesStringColumn()
}
return []sqltypes.Value{}, nil
}
// valueToBigRat converts a numeric 'value' regarded as having type 'valueType' into a
// big.Rat object.
// Note:
// We use an explicit valueType rather than depend on the type stored in 'value' to force
// the type of MAX(column) or MIN(column) to correspond to the type of 'column'.
// (We've had issues where the type of MAX(column) returned by Vitess was signed even if the
// type of column was unsigned).
func valueToBigRat(value sqltypes.Value, valueType querypb.Type) (*big.Rat, error) {
switch {
case sqltypes.IsUnsigned(valueType):
nativeValue, err := value.ParseUint64()
if err != nil {
return nil, err
}
return uint64ToBigRat(nativeValue), nil
case sqltypes.IsSigned(valueType):
nativeValue, err := value.ParseInt64()
if err != nil {
return nil, err
}
return int64ToBigRat(nativeValue), nil
case sqltypes.IsFloat(valueType):
nativeValue, err := value.ParseFloat64()
if err != nil {
return nil, err
}
return float64ToBigRat(nativeValue), nil
default:
panic(fmt.Sprintf("got value with a non numeric type: %v", value))
}
}
// FindChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func FindChunks(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, td *tabletmanagerdatapb.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
result := []string{"", ""}
// eliminate a few cases we don't split tables for
if len(td.PrimaryKeyColumns) == 0 {
// no primary key, what can we do?
return result, nil
}
if td.DataLength < minTableSizeForSplit {
// table is too small to split up
return result, nil
}
// get the min and max of the leading column of the primary key
query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, ti, query, 1, true)
cancel()
if err != nil {
return nil, fmt.Errorf("ExecuteFetchAsApp: %v", err)
}
if len(qr.Rows) != 1 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
return result, nil
}
if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() {
wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1])
return result, nil
}
switch {
case sqltypes.IsSigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
min, err := minNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / int64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := int64(1); i < int64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsUnsigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
min, err := minNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / uint64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := uint64(1); i < uint64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsFloat(qr.Fields[0].Type):
min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err)
return result, nil
}
max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err)
return result, nil
}
interval := (max - min) / float64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := 1; i < sourceReaderCount; i++ {
result[i] = fmt.Sprintf("%v", min+interval*float64(i))
}
return result, nil
}
wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
return result, nil
}
// FindChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func FindChunks(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, td *tabletmanagerdatapb.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
result := []string{"", ""}
// eliminate a few cases we don't split tables for
if len(td.PrimaryKeyColumns) == 0 {
// no primary key, what can we do?
return result, nil
}
if td.DataLength < minTableSizeForSplit {
// table is too small to split up
return result, nil
}
// get the min and max of the leading column of the primary key
query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, ti, query, 1, true)
cancel()
if err != nil {
return nil, fmt.Errorf("ExecuteFetchAsApp: %v", err)
}
if len(qr.Rows) != 1 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
return result, nil
}
// FIXME(alainjobart) this code is a bit clunky. I'd like to
// convert the first row into an array of Values, and then
// see which type they are and go from there. Can only happen after
// Value has a full type.
l0 := qr.Rows[0].Lengths[0]
l1 := qr.Rows[0].Lengths[1]
if l0 < 0 || l1 < 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v", td.Name, qr.Rows[0])
return result, nil
}
minValue := qr.Rows[0].Values[:l0]
maxValue := qr.Rows[0].Values[l0 : l0+l1]
switch {
case sqltypes.IsSigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(minValue)
maxNumeric := sqltypes.MakeNumeric(maxValue)
min, err := minNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / int64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := int64(1); i < int64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsUnsigned(qr.Fields[0].Type):
minNumeric := sqltypes.MakeNumeric(minValue)
maxNumeric := sqltypes.MakeNumeric(maxValue)
min, err := minNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / uint64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := uint64(1); i < uint64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case sqltypes.IsFloat(qr.Fields[0].Type):
min, err := strconv.ParseFloat(string(minValue), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, string(minValue), err)
return result, nil
}
max, err := strconv.ParseFloat(string(maxValue), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, string(maxValue), err)
return result, nil
}
interval := (max - min) / float64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := 1; i < sourceReaderCount; i++ {
result[i] = fmt.Sprintf("%v", min+interval*float64(i))
}
return result, nil
}
wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
return result, nil
}