func checkEquivExpr(a, b parser.TypedExpr, qvals qvalMap) error {
// The expressions above only use the values 1 and 2. Verify that the
// simplified expressions evaluate to the same value as the original
// expression for interesting values.
for _, v := range []parser.Datum{
parser.NewDInt(0),
parser.NewDInt(1),
parser.NewDInt(2),
parser.NewDInt(3),
parser.DNull,
} {
for _, q := range qvals {
q.datum = v
}
da, err := a.Eval(parser.EvalContext{})
if err != nil {
return fmt.Errorf("%s: %v", a, err)
}
db, err := b.Eval(parser.EvalContext{})
if err != nil {
return fmt.Errorf("%s: %v", b, err)
}
// This is tricky: we don't require the expressions to produce identical
// results, but to either both return true or both return not true (either
// false or NULL).
if (da == parser.DBoolTrue) != (db == parser.DBoolTrue) {
return fmt.Errorf("%s: %s: expected %s, but found %s", a, v, da, db)
}
}
return nil
}
// golangFillQueryArguments populates the placeholder map with
// types and values from an array of Go values.
// TODO: This does not support arguments of the SQL 'Date' type, as there is not
// an equivalent type in Go's standard library. It's not currently needed by any
// of our internal tables.
func golangFillQueryArguments(pinfo *parser.PlaceholderInfo, args []interface{}) {
pinfo.Clear()
for i, arg := range args {
k := fmt.Sprint(i + 1)
if arg == nil {
pinfo.SetValue(k, parser.DNull)
continue
}
// A type switch to handle a few explicit types with special semantics:
// - Datums are passed along as is.
// - Time datatypes get special representation in the database.
var d parser.Datum
switch t := arg.(type) {
case parser.Datum:
d = t
case time.Time:
d = parser.MakeDTimestamp(t, time.Microsecond)
case time.Duration:
d = &parser.DInterval{Duration: duration.Duration{Nanos: t.Nanoseconds()}}
case *inf.Dec:
dd := &parser.DDecimal{}
dd.Set(t)
d = dd
}
if d == nil {
// Handle all types which have an underlying type that can be stored in the
// database.
// Note: if this reflection becomes a performance concern in the future,
// commonly used types could be added explicitly into the type switch above
// for a performance gain.
val := reflect.ValueOf(arg)
switch val.Kind() {
case reflect.Bool:
d = parser.MakeDBool(parser.DBool(val.Bool()))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
d = parser.NewDInt(parser.DInt(val.Int()))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
d = parser.NewDInt(parser.DInt(val.Uint()))
case reflect.Float32, reflect.Float64:
d = parser.NewDFloat(parser.DFloat(val.Float()))
case reflect.String:
d = parser.NewDString(val.String())
case reflect.Slice:
// Handle byte slices.
if val.Type().Elem().Kind() == reflect.Uint8 {
d = parser.NewDBytes(parser.DBytes(val.Bytes()))
}
}
if d == nil {
panic(fmt.Sprintf("unexpected type %T", arg))
}
}
pinfo.SetValue(k, d)
}
}
// Arg implements the parser.Args interface.
// TODO: This does not support arguments of the SQL 'Date' type, as there is not
// an equivalent type in Go's standard library. It's not currently needed by any
// of our internal tables.
func (gp golangParameters) Arg(name string) (parser.Datum, bool) {
i, err := processPositionalArgument(name)
if err != nil {
return nil, false
}
if i < 1 || int(i) > len(gp) {
return nil, false
}
arg := gp[i-1]
if arg == nil {
return parser.DNull, true
}
// A type switch to handle a few explicit types with special semantics.
switch t := arg.(type) {
// Datums are passed along as is.
case parser.Datum:
return t, true
// Time datatypes get special representation in the database.
case time.Time:
return parser.MakeDTimestamp(t, time.Microsecond), true
case time.Duration:
return &parser.DInterval{Duration: duration.Duration{Nanos: t.Nanoseconds()}}, true
case *inf.Dec:
dd := &parser.DDecimal{}
dd.Set(t)
return dd, true
}
// Handle all types which have an underlying type that can be stored in the
// database.
// Note: if this reflection becomes a performance concern in the future,
// commonly used types could be added explicitly into the type switch above
// for a performance gain.
val := reflect.ValueOf(arg)
switch val.Kind() {
case reflect.Bool:
return parser.MakeDBool(parser.DBool(val.Bool())), true
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return parser.NewDInt(parser.DInt(val.Int())), true
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return parser.NewDInt(parser.DInt(val.Uint())), true
case reflect.Float32, reflect.Float64:
return parser.NewDFloat(parser.DFloat(val.Float())), true
case reflect.String:
return parser.NewDString(val.String()), true
case reflect.Slice:
// Handle byte slices.
if val.Type().Elem().Kind() == reflect.Uint8 {
return parser.NewDBytes(parser.DBytes(val.Bytes())), true
}
}
panic(fmt.Sprintf("unexpected type %T", arg))
}
// Events is an endpoint that returns the latest event log entries, with the following
// optional URL parameters:
//
// type=STRING returns events with this type (e.g. "create_table")
// targetID=INT returns events for that have this targetID
func (s *adminServer) Events(ctx context.Context, req *serverpb.EventsRequest) (*serverpb.EventsResponse, error) {
args := sql.SessionArgs{User: s.getUser(req)}
session := sql.NewSession(ctx, args, s.server.sqlExecutor, nil)
// Execute the query.
q := makeSQLQuery()
q.Append("SELECT timestamp, eventType, targetID, reportingID, info, uniqueID ")
q.Append("FROM system.eventlog ")
q.Append("WHERE true ") // This simplifies the WHERE clause logic below.
if len(req.Type) > 0 {
q.Append("AND eventType = $ ", parser.NewDString(req.Type))
}
if req.TargetId > 0 {
q.Append("AND targetID = $ ", parser.NewDInt(parser.DInt(req.TargetId)))
}
q.Append("ORDER BY timestamp DESC ")
q.Append("LIMIT $", parser.NewDInt(parser.DInt(apiEventLimit)))
if len(q.Errors()) > 0 {
return nil, s.serverErrors(q.Errors())
}
r := s.server.sqlExecutor.ExecuteStatements(session, q.String(), q.QueryArguments())
if err := s.checkQueryResults(r.ResultList, 1); err != nil {
return nil, s.serverError(err)
}
// Marshal response.
var resp serverpb.EventsResponse
scanner := makeResultScanner(r.ResultList[0].Columns)
for _, row := range r.ResultList[0].Rows {
var event serverpb.EventsResponse_Event
var ts time.Time
if err := scanner.ScanIndex(row, 0, &ts); err != nil {
return nil, err
}
event.Timestamp = serverpb.EventsResponse_Event_Timestamp{Sec: ts.Unix(), Nsec: uint32(ts.Nanosecond())}
if err := scanner.ScanIndex(row, 1, &event.EventType); err != nil {
return nil, err
}
if err := scanner.ScanIndex(row, 2, &event.TargetID); err != nil {
return nil, err
}
if err := scanner.ScanIndex(row, 3, &event.ReportingID); err != nil {
return nil, err
}
if err := scanner.ScanIndex(row, 4, &event.Info); err != nil {
return nil, err
}
if err := scanner.ScanIndex(row, 5, &event.UniqueID); err != nil {
return nil, err
}
resp.Events = append(resp.Events, event)
}
return &resp, nil
}
// queryNamespaceID queries for the ID of the namespace with the given name and
// parent ID.
func (s *adminServer) queryNamespaceID(
session *sql.Session, parentID sqlbase.ID, name string,
) (sqlbase.ID, error) {
const query = `SELECT id FROM system.namespace WHERE parentID = $1 AND name = $2`
params := parser.NewPlaceholderInfo()
params.SetValue(`1`, parser.NewDInt(parser.DInt(parentID)))
params.SetValue(`2`, parser.NewDString(name))
r := s.server.sqlExecutor.ExecuteStatements(session, query, params)
if err := s.checkQueryResults(r.ResultList, 1); err != nil {
return 0, err
}
result := r.ResultList[0]
if len(result.Rows) == 0 {
return 0, errors.Errorf("namespace %s with ParentID %d not found", name, parentID)
}
var id int64
scanner := resultScanner{}
err := scanner.ScanIndex(result.Rows[0], 0, &id)
if err != nil {
return 0, err
}
return sqlbase.ID(id), nil
}
// queryZone retrieves the specific ZoneConfig associated with the supplied ID,
// if it exists.
func (s *adminServer) queryZone(
session *sql.Session, id sqlbase.ID,
) (config.ZoneConfig, bool, error) {
const query = `SELECT config FROM system.zones WHERE id = $1`
params := parser.NewPlaceholderInfo()
params.SetValue(`1`, parser.NewDInt(parser.DInt(id)))
r := s.server.sqlExecutor.ExecuteStatements(session, query, params)
if err := s.checkQueryResults(r.ResultList, 1); err != nil {
return config.ZoneConfig{}, false, err
}
result := r.ResultList[0]
if len(result.Rows) == 0 {
return config.ZoneConfig{}, false, nil
}
var zoneBytes []byte
scanner := resultScanner{}
err := scanner.ScanIndex(result.Rows[0], 0, &zoneBytes)
if err != nil {
return config.ZoneConfig{}, false, err
}
var zone config.ZoneConfig
if err := zone.Unmarshal(zoneBytes); err != nil {
return config.ZoneConfig{}, false, err
}
return zone, true, nil
}
func makeIntTestDatum(count int) []parser.Datum {
rng, _ := randutil.NewPseudoRand()
vals := make([]parser.Datum, count)
for i := range vals {
vals[i] = parser.NewDInt(parser.DInt(rng.Int63()))
}
return vals
}
// ShowIndex returns all the indexes for a table.
// Privileges: Any privilege on table.
// Notes: postgres does not have a SHOW INDEXES statement.
// mysql requires some privilege for any column.
func (p *planner) ShowIndex(n *parser.ShowIndex) (planNode, error) {
tn, err := n.Table.NormalizeWithDatabaseName(p.session.Database)
if err != nil {
return nil, err
}
desc, err := p.mustGetTableDesc(tn)
if err != nil {
return nil, err
}
if err := p.anyPrivilege(desc); err != nil {
return nil, err
}
v := &valuesNode{
columns: []ResultColumn{
{Name: "Table", Typ: parser.TypeString},
{Name: "Name", Typ: parser.TypeString},
{Name: "Unique", Typ: parser.TypeBool},
{Name: "Seq", Typ: parser.TypeInt},
{Name: "Column", Typ: parser.TypeString},
{Name: "Direction", Typ: parser.TypeString},
{Name: "Storing", Typ: parser.TypeBool},
},
}
appendRow := func(index sqlbase.IndexDescriptor, colName string, sequence int,
direction string, isStored bool) {
v.rows = append(v.rows, []parser.Datum{
parser.NewDString(tn.Table()),
parser.NewDString(index.Name),
parser.MakeDBool(parser.DBool(index.Unique)),
parser.NewDInt(parser.DInt(sequence)),
parser.NewDString(colName),
parser.NewDString(direction),
parser.MakeDBool(parser.DBool(isStored)),
})
}
for _, index := range append([]sqlbase.IndexDescriptor{desc.PrimaryIndex}, desc.Indexes...) {
sequence := 1
for i, col := range index.ColumnNames {
appendRow(index, col, sequence, index.ColumnDirections[i].String(), false)
sequence++
}
for _, col := range index.StoreColumnNames {
appendRow(index, col, sequence, "N/A", true)
sequence++
}
}
return v, nil
}
// RandDatum generates a random Datum of the given type.
// If null is true, the datum can be DNull.
func RandDatum(rng *rand.Rand, typ ColumnType_Kind, null bool) parser.Datum {
if null && rng.Intn(10) == 0 {
return parser.DNull
}
switch typ {
case ColumnType_BOOL:
return parser.MakeDBool(rng.Intn(2) == 1)
case ColumnType_INT:
return parser.NewDInt(parser.DInt(rng.Int63()))
case ColumnType_FLOAT:
return parser.NewDFloat(parser.DFloat(rng.NormFloat64()))
case ColumnType_DECIMAL:
d := &parser.DDecimal{}
d.Dec.SetScale(inf.Scale(rng.Intn(40) - 20))
d.Dec.SetUnscaled(rng.Int63())
return d
case ColumnType_DATE:
return parser.NewDDate(parser.DDate(rng.Intn(10000)))
case ColumnType_TIMESTAMP:
return &parser.DTimestamp{Time: time.Unix(rng.Int63n(1000000), rng.Int63n(1000000))}
case ColumnType_INTERVAL:
return &parser.DInterval{Duration: duration.Duration{Months: rng.Int63n(1000),
Days: rng.Int63n(1000),
Nanos: rng.Int63n(1000000),
}}
case ColumnType_STRING:
// Generate a random ASCII string.
p := make([]byte, rng.Intn(10))
for i := range p {
p[i] = byte(1 + rng.Intn(127))
}
return parser.NewDString(string(p))
case ColumnType_BYTES:
p := make([]byte, rng.Intn(10))
_, _ = rng.Read(p)
return parser.NewDBytes(parser.DBytes(p))
case ColumnType_TIMESTAMPTZ:
return &parser.DTimestampTZ{Time: time.Unix(rng.Int63n(1000000), rng.Int63n(1000000))}
default:
panic(fmt.Sprintf("invalid type %s", typ))
}
}
// decodeOidDatum decodes bytes with specified Oid and format code into
// a datum.
func decodeOidDatum(id oid.Oid, code formatCode, b []byte) (parser.Datum, error) {
var d parser.Datum
switch id {
case oid.T_bool:
switch code {
case formatText:
v, err := strconv.ParseBool(string(b))
if err != nil {
return d, err
}
d = parser.MakeDBool(parser.DBool(v))
default:
return d, fmt.Errorf("unsupported bool format code: %d", code)
}
case oid.T_int2:
switch code {
case formatText:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
case formatBinary:
var i int16
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
default:
return d, fmt.Errorf("unsupported int2 format code: %d", code)
}
case oid.T_int4:
switch code {
case formatText:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
case formatBinary:
var i int32
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
default:
return d, fmt.Errorf("unsupported int4 format code: %d", code)
}
case oid.T_int8:
switch code {
case formatText:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
case formatBinary:
var i int64
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
default:
return d, fmt.Errorf("unsupported int8 format code: %d", code)
}
case oid.T_float4:
switch code {
case formatText:
f, err := strconv.ParseFloat(string(b), 64)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
case formatBinary:
var f float32
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
default:
return d, fmt.Errorf("unsupported float4 format code: %d", code)
}
case oid.T_float8:
switch code {
case formatText:
f, err := strconv.ParseFloat(string(b), 64)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
case formatBinary:
var f float64
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
default:
return d, fmt.Errorf("unsupported float8 format code: %d", code)
}
case oid.T_numeric:
switch code {
case formatText:
dd := &parser.DDecimal{}
if _, ok := dd.SetString(string(b)); !ok {
return nil, fmt.Errorf("could not parse string %q as decimal", b)
}
d = dd
default:
return d, fmt.Errorf("unsupported numeric format code: %d", code)
}
case oid.T_text, oid.T_varchar:
switch code {
case formatText:
d = parser.NewDString(string(b))
default:
return d, fmt.Errorf("unsupported text format code: %d", code)
}
case oid.T_bytea:
switch code {
case formatText:
// http://www.postgresql.org/docs/current/static/datatype-binary.html#AEN5667
// Code cribbed from github.com/lib/pq.
// We only support hex encoding.
if len(b) >= 2 && bytes.Equal(b[:2], []byte("\\x")) {
b = b[2:] // trim off leading "\\x"
result := make([]byte, hex.DecodedLen(len(b)))
_, err := hex.Decode(result, b)
if err != nil {
return d, err
}
d = parser.NewDBytes(parser.DBytes(result))
} else {
return d, fmt.Errorf("unsupported bytea encoding: %q", b)
}
case formatBinary:
d = parser.NewDBytes(parser.DBytes(b))
default:
return d, fmt.Errorf("unsupported bytea format code: %d", code)
}
case oid.T_timestamp, oid.T_timestamptz:
switch code {
case formatText:
ts, err := parseTs(string(b))
if err != nil {
return d, fmt.Errorf("could not parse string %q as timestamp", b)
}
d = &parser.DTimestamp{Time: ts}
case formatBinary:
return d, fmt.Errorf("unsupported timestamp format code: %d", code)
}
case oid.T_date:
switch code {
case formatText:
ts, err := parseTs(string(b))
if err != nil {
return d, fmt.Errorf("could not parse string %q as date", b)
}
daysSinceEpoch := ts.Unix() / secondsInDay
d = parser.NewDDate(parser.DDate(daysSinceEpoch))
case formatBinary:
return d, fmt.Errorf("unsupported date format code: %d", code)
}
default:
return d, fmt.Errorf("unsupported OID: %v", id)
}
return d, nil
}
// decodeOidDatum decodes bytes with specified Oid and format code into
// a datum.
func decodeOidDatum(id oid.Oid, code formatCode, b []byte) (parser.Datum, error) {
var d parser.Datum
switch id {
case oid.T_bool:
switch code {
case formatText:
v, err := strconv.ParseBool(string(b))
if err != nil {
return d, err
}
d = parser.MakeDBool(parser.DBool(v))
case formatBinary:
switch b[0] {
case 0:
d = parser.MakeDBool(false)
case 1:
d = parser.MakeDBool(true)
default:
return d, util.Errorf("unsupported binary bool: %q", b)
}
default:
return d, util.Errorf("unsupported bool format code: %s", code)
}
case oid.T_int2:
switch code {
case formatText:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
case formatBinary:
var i int16
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
default:
return d, util.Errorf("unsupported int2 format code: %s", code)
}
case oid.T_int4:
switch code {
case formatText:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
case formatBinary:
var i int32
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
default:
return d, util.Errorf("unsupported int4 format code: %s", code)
}
case oid.T_int8:
switch code {
case formatText:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
case formatBinary:
var i int64
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i)
if err != nil {
return d, err
}
d = parser.NewDInt(parser.DInt(i))
default:
return d, util.Errorf("unsupported int8 format code: %s", code)
}
case oid.T_float4:
switch code {
case formatText:
f, err := strconv.ParseFloat(string(b), 64)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
case formatBinary:
var f float32
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
default:
return d, util.Errorf("unsupported float4 format code: %s", code)
}
case oid.T_float8:
switch code {
case formatText:
f, err := strconv.ParseFloat(string(b), 64)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
case formatBinary:
var f float64
err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f)
if err != nil {
return d, err
}
d = parser.NewDFloat(parser.DFloat(f))
default:
return d, util.Errorf("unsupported float8 format code: %s", code)
}
case oid.T_numeric:
switch code {
case formatText:
dd := &parser.DDecimal{}
if _, ok := dd.SetString(string(b)); !ok {
return nil, util.Errorf("could not parse string %q as decimal", b)
}
d = dd
case formatBinary:
r := bytes.NewReader(b)
alloc := struct {
pgNum pgNumeric
i16 int16
dd parser.DDecimal
}{}
for _, ptr := range []interface{}{
&alloc.pgNum.ndigits,
&alloc.pgNum.weight,
&alloc.pgNum.sign,
&alloc.pgNum.dscale,
} {
if err := binary.Read(r, binary.BigEndian, ptr); err != nil {
return d, err
}
}
decDigits := make([]byte, 0, alloc.pgNum.ndigits*pgDecDigits)
for i := int16(0); i < alloc.pgNum.ndigits-1; i++ {
if err := binary.Read(r, binary.BigEndian, &alloc.i16); err != nil {
return d, err
}
decDigits = strconv.AppendUint(decDigits, uint64(alloc.i16), 10)
}
// The last digit may contain padding, which we need to deal with.
if err := binary.Read(r, binary.BigEndian, &alloc.i16); err != nil {
return d, err
}
dscale := (alloc.pgNum.ndigits - 1 - alloc.pgNum.weight) * pgDecDigits
if overScale := dscale - alloc.pgNum.dscale; overScale > 0 {
dscale -= overScale
for i := int16(0); i < overScale; i++ {
alloc.i16 /= 10
}
}
decDigits = strconv.AppendUint(decDigits, uint64(alloc.i16), 10)
alloc.dd.UnscaledBig().SetString(string(decDigits), 10)
alloc.dd.SetScale(inf.Scale(dscale))
switch alloc.pgNum.sign {
case pgNumericPos:
case pgNumericNeg:
alloc.dd.Neg(&alloc.dd.Dec)
default:
return d, util.Errorf("unsupported numeric sign: %s", alloc.pgNum.sign)
}
d = &alloc.dd
default:
return d, util.Errorf("unsupported numeric format code: %s", code)
}
case oid.T_text, oid.T_varchar:
switch code {
case formatText, formatBinary:
d = parser.NewDString(string(b))
default:
return d, util.Errorf("unsupported text format code: %s", code)
}
case oid.T_bytea:
switch code {
case formatText:
// http://www.postgresql.org/docs/current/static/datatype-binary.html#AEN5667
// Code cribbed from github.com/lib/pq.
// We only support hex encoding.
if len(b) >= 2 && bytes.Equal(b[:2], []byte("\\x")) {
b = b[2:] // trim off leading "\\x"
result := make([]byte, hex.DecodedLen(len(b)))
_, err := hex.Decode(result, b)
if err != nil {
return d, err
}
d = parser.NewDBytes(parser.DBytes(result))
} else {
return d, util.Errorf("unsupported bytea encoding: %q", b)
}
case formatBinary:
d = parser.NewDBytes(parser.DBytes(b))
default:
return d, util.Errorf("unsupported bytea format code: %s", code)
}
case oid.T_timestamp, oid.T_timestamptz:
switch code {
case formatText:
ts, err := parseTs(string(b))
if err != nil {
return d, util.Errorf("could not parse string %q as timestamp", b)
}
d = parser.MakeDTimestamp(ts, time.Microsecond)
default:
return d, util.Errorf("unsupported timestamp format code: %s", code)
}
case oid.T_date:
switch code {
case formatText:
ts, err := parseTs(string(b))
if err != nil {
return d, util.Errorf("could not parse string %q as date", b)
}
daysSinceEpoch := ts.Unix() / secondsInDay
d = parser.NewDDate(parser.DDate(daysSinceEpoch))
default:
return d, util.Errorf("unsupported date format code: %s", code)
}
default:
return d, util.Errorf("unsupported OID: %v", id)
}
return d, nil
}
func dumpTable(w io.Writer, conn *sqlConn, origDBName, origTableName string) error {
const limit = 100
// Escape names since they can't be used in placeholders.
dbname := parser.Name(origDBName).String()
tablename := parser.Name(origTableName).String()
if err := conn.Exec(fmt.Sprintf("SET DATABASE = %s", dbname), nil); err != nil {
return err
}
// Fetch all table metadata in a transaction and its time to guarantee it
// doesn't change between the various SHOW statements.
if err := conn.Exec("BEGIN", nil); err != nil {
return err
}
vals, err := conn.QueryRow("SELECT cluster_logical_timestamp()::int", nil)
if err != nil {
return err
}
clusterTSStart := vals[0].(int64)
clusterTS := time.Unix(0, clusterTSStart).Format(time.RFC3339Nano)
// Fetch table descriptor.
vals, err = conn.QueryRow(`
SELECT descriptor
FROM system.descriptor
JOIN system.namespace tables
ON tables.id = descriptor.id
JOIN system.namespace dbs
ON dbs.id = tables.parentid
WHERE tables.name = $1 AND dbs.name = $2`,
[]driver.Value{origTableName, origDBName})
if err == io.EOF {
return errors.Errorf("unknown database or table %s.%s", origTableName, origDBName)
} else if err != nil {
return err
}
b := vals[0].([]byte)
var desc sqlbase.Descriptor
if err := proto.Unmarshal(b, &desc); err != nil {
return err
}
table := desc.GetTable()
if table == nil {
return errors.New("internal error: expected table descriptor")
}
coltypes := make(map[string]string)
for _, c := range table.Columns {
coltypes[c.Name] = c.Type.SQLString()
}
primaryIndex := table.PrimaryIndex.Name
index := table.PrimaryIndex.ColumnNames
indexes := strings.Join(index, ", ")
// Build the SELECT query.
var sbuf bytes.Buffer
fmt.Fprintf(&sbuf, "SELECT %s, * FROM %s@%s AS OF SYSTEM TIME '%s'", indexes, tablename, primaryIndex, clusterTS)
var wbuf bytes.Buffer
fmt.Fprintf(&wbuf, " WHERE ROW (%s) > ROW (", indexes)
for i := range index {
if i > 0 {
wbuf.WriteString(", ")
}
fmt.Fprintf(&wbuf, "$%d", i+1)
}
wbuf.WriteString(")")
// No WHERE clause first time, so add a place to inject it.
fmt.Fprintf(&sbuf, "%%s ORDER BY %s LIMIT %d", indexes, limit)
bs := sbuf.String()
vals, err = conn.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", tablename), nil)
if err != nil {
return err
}
create := vals[1].([]byte)
if _, err := w.Write(create); err != nil {
return err
}
if _, err := w.Write([]byte(";\n")); err != nil {
return err
}
if err := conn.Exec("COMMIT", nil); err != nil {
return err
}
// pk holds the last values of the fetched primary keys
var pk []driver.Value
q := fmt.Sprintf(bs, "")
for {
rows, err := conn.Query(q, pk)
if err != nil {
return err
}
cols := rows.Columns()
pkcols := cols[:len(index)]
cols = cols[len(index):]
inserts := make([][]string, 0, limit)
i := 0
for i < limit {
vals := make([]driver.Value, len(cols)+len(pkcols))
if err := rows.Next(vals); err == io.EOF {
break
} else if err != nil {
return err
}
if pk == nil {
q = fmt.Sprintf(bs, wbuf.String())
}
pk = vals[:len(index)]
vals = vals[len(index):]
ivals := make([]string, len(vals))
// Values need to be correctly encoded for INSERT statements in a text file.
for si, sv := range vals {
switch t := sv.(type) {
case nil:
ivals[si] = "NULL"
case bool:
ivals[si] = parser.MakeDBool(parser.DBool(t)).String()
case int64:
ivals[si] = parser.NewDInt(parser.DInt(t)).String()
case float64:
ivals[si] = parser.NewDFloat(parser.DFloat(t)).String()
case []byte:
switch ct := coltypes[cols[si]]; ct {
case "INTERVAL":
ivals[si] = fmt.Sprintf("'%s'", t)
case "DECIMAL":
ivals[si] = fmt.Sprintf("%s", t)
default:
// STRING and BYTES types can have optional length suffixes, so only examine
// the prefix of the type.
if strings.HasPrefix(coltypes[cols[si]], "STRING") {
ivals[si] = parser.NewDString(string(t)).String()
} else if strings.HasPrefix(coltypes[cols[si]], "BYTES") {
ivals[si] = parser.NewDBytes(parser.DBytes(t)).String()
} else {
panic(errors.Errorf("unknown []byte type: %s, %v: %s", t, cols[si], coltypes[cols[si]]))
}
}
case time.Time:
var d parser.Datum
ct := coltypes[cols[si]]
switch ct {
case "DATE":
d = parser.NewDDateFromTime(t, time.UTC)
case "TIMESTAMP":
d = parser.MakeDTimestamp(t, time.Nanosecond)
case "TIMESTAMP WITH TIME ZONE":
d = parser.MakeDTimestampTZ(t, time.Nanosecond)
default:
panic(errors.Errorf("unknown timestamp type: %s, %v: %s", t, cols[si], coltypes[cols[si]]))
}
ivals[si] = fmt.Sprintf("'%s'", d)
default:
panic(errors.Errorf("unknown field type: %T (%s)", t, cols[si]))
}
}
inserts = append(inserts, ivals)
i++
}
for si, sv := range pk {
b, ok := sv.([]byte)
if ok && strings.HasPrefix(coltypes[pkcols[si]], "STRING") {
// Primary key strings need to be converted to a go string, but not SQL
// encoded since they aren't being written to a text file.
pk[si] = string(b)
}
}
if err := rows.Close(); err != nil {
return err
}
if i == 0 {
break
}
fmt.Fprintf(w, "\nINSERT INTO %s VALUES", tablename)
for idx, values := range inserts {
if idx > 0 {
fmt.Fprint(w, ",")
}
fmt.Fprint(w, "\n\t(")
for vi, v := range values {
if vi > 0 {
fmt.Fprint(w, ", ")
}
fmt.Fprint(w, v)
}
fmt.Fprint(w, ")")
}
fmt.Fprintln(w, ";")
if i < limit {
break
}
}
return nil
}
func TestIndexKey(t *testing.T) {
rng, _ := randutil.NewPseudoRand()
var a DatumAlloc
tests := []indexKeyTest{
{nil, nil,
[]parser.Datum{parser.NewDInt(10)},
[]parser.Datum{parser.NewDInt(20)},
},
{[]ID{100}, nil,
[]parser.Datum{parser.NewDInt(10), parser.NewDInt(11)},
[]parser.Datum{parser.NewDInt(20)},
},
{[]ID{100, 200}, nil,
[]parser.Datum{parser.NewDInt(10), parser.NewDInt(11), parser.NewDInt(12)},
[]parser.Datum{parser.NewDInt(20)},
},
{nil, []ID{100},
[]parser.Datum{parser.NewDInt(10)},
[]parser.Datum{parser.NewDInt(20), parser.NewDInt(21)},
},
{[]ID{100}, []ID{100},
[]parser.Datum{parser.NewDInt(10), parser.NewDInt(11)},
[]parser.Datum{parser.NewDInt(20), parser.NewDInt(21)},
},
{[]ID{100}, []ID{200},
[]parser.Datum{parser.NewDInt(10), parser.NewDInt(11)},
[]parser.Datum{parser.NewDInt(20), parser.NewDInt(21)},
},
{[]ID{100, 200}, []ID{100, 300},
[]parser.Datum{parser.NewDInt(10), parser.NewDInt(11), parser.NewDInt(12)},
[]parser.Datum{parser.NewDInt(20), parser.NewDInt(21), parser.NewDInt(22)},
},
}
for i := 0; i < 1000; i++ {
var t indexKeyTest
t.primaryInterleaves = make([]ID, rng.Intn(10))
for j := range t.primaryInterleaves {
t.primaryInterleaves[j] = ID(1 + rng.Intn(10))
}
valuesLen := randutil.RandIntInRange(rng, len(t.primaryInterleaves)+1, len(t.primaryInterleaves)+10)
t.primaryValues = make([]parser.Datum, valuesLen)
for j := range t.primaryValues {
t.primaryValues[j] = RandDatum(rng, ColumnType_INT, true)
}
t.secondaryInterleaves = make([]ID, rng.Intn(10))
for j := range t.secondaryInterleaves {
t.secondaryInterleaves[j] = ID(1 + rng.Intn(10))
}
valuesLen = randutil.RandIntInRange(rng, len(t.secondaryInterleaves)+1, len(t.secondaryInterleaves)+10)
t.secondaryValues = make([]parser.Datum, valuesLen)
for j := range t.secondaryValues {
t.secondaryValues[j] = RandDatum(rng, ColumnType_INT, true)
}
tests = append(tests, t)
}
for i, test := range tests {
tableDesc, colMap := makeTableDescForTest(test)
testValues := append(test.primaryValues, test.secondaryValues...)
primaryKeyPrefix := MakeIndexKeyPrefix(&tableDesc, tableDesc.PrimaryIndex.ID)
primaryKey, _, err := EncodeIndexKey(
&tableDesc, &tableDesc.PrimaryIndex, colMap, testValues, primaryKeyPrefix)
if err != nil {
t.Fatal(err)
}
primaryValue := roachpb.MakeValueFromBytes(nil)
primaryIndexKV := client.KeyValue{Key: primaryKey, Value: &primaryValue}
secondaryIndexEntry, err := EncodeSecondaryIndex(
&tableDesc, &tableDesc.Indexes[0], colMap, testValues)
if err != nil {
t.Fatal(err)
}
secondaryIndexKV := client.KeyValue{
Key: secondaryIndexEntry.Key,
Value: &secondaryIndexEntry.Value,
}
checkEntry := func(index *IndexDescriptor, entry client.KeyValue) {
values, err := decodeIndex(&a, &tableDesc, index, entry.Key)
if err != nil {
t.Fatal(err)
}
for j, value := range values {
testValue := testValues[colMap[index.ColumnIDs[j]]]
if value.Compare(testValue) != 0 {
t.Fatalf("%d: value %d got %q but expected %q", i, j, value, testValue)
}
}
indexID, _, err := DecodeIndexKeyPrefix(&a, &tableDesc, entry.Key)
if err != nil {
t.Fatal(err)
}
if indexID != index.ID {
t.Errorf("%d", i)
}
extracted, err := ExtractIndexKey(&a, &tableDesc, entry)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(extracted, primaryKey) {
t.Errorf("%d got %s <%x>, but expected %s <%x>", i, extracted, []byte(extracted), roachpb.Key(primaryKey), primaryKey)
}
}
checkEntry(&tableDesc.PrimaryIndex, primaryIndexKV)
checkEntry(&tableDesc.Indexes[0], secondaryIndexKV)
}
}
func TestTableReader(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sqlDB, kvDB, cleanup := sqlutils.SetupServer(t)
defer cleanup()
// Create a table where each row is:
//
// | a | b | sum | s |
// |-----------------------------------------------------------------|
// | rowId/10 | rowId%10 | rowId/10 + rowId%10 | IntToEnglish(rowId) |
aFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row / 10))
}
bFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row % 10))
}
sumFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row/10 + row%10))
}
sqlutils.CreateTable(t, sqlDB, "t",
"a INT, b INT, sum INT, s STRING, PRIMARY KEY (a,b), INDEX bs (b,s)",
99,
sqlutils.ToRowFn(aFn, bFn, sumFn, sqlutils.RowEnglishFn))
td := sqlbase.GetTableDescriptor(kvDB, "test", "t")
makeIndexSpan := func(start, end int) TableReaderSpan {
var span roachpb.Span
prefix := roachpb.Key(sqlbase.MakeIndexKeyPrefix(td.ID, td.Indexes[0].ID))
span.Key = append(prefix, encoding.EncodeVarintAscending(nil, int64(start))...)
span.EndKey = append(span.EndKey, prefix...)
span.EndKey = append(span.EndKey, encoding.EncodeVarintAscending(nil, int64(end))...)
return TableReaderSpan{Span: span}
}
testCases := []struct {
spec TableReaderSpec
expected string
}{
{
spec: TableReaderSpec{
Filter: Expression{Expr: "$2 < 5 AND $1 != 3"}, // sum < 5 && b != 3
OutputColumns: []uint32{0, 1},
},
expected: "[[0 1] [0 2] [0 4] [1 0] [1 1] [1 2] [2 0] [2 1] [2 2] [3 0] [3 1] [4 0]]",
},
{
spec: TableReaderSpec{
Filter: Expression{Expr: "$2 < 5 AND $1 != 3"},
OutputColumns: []uint32{3}, // s
HardLimit: 4,
},
expected: "[['one'] ['two'] ['four'] ['one-zero']]",
},
{
spec: TableReaderSpec{
IndexIdx: 1,
Reverse: true,
Spans: []TableReaderSpan{makeIndexSpan(4, 6)},
Filter: Expression{Expr: "$0 < 3"}, // sum < 8
OutputColumns: []uint32{0, 1},
SoftLimit: 1,
},
expected: "[[2 5] [1 5] [0 5] [2 4] [1 4] [0 4]]",
},
}
for _, c := range testCases {
ts := c.spec
ts.Table = *td
txn := client.NewTxn(context.Background(), *kvDB)
out := &RowBuffer{}
tr, err := newTableReader(&ts, txn, out, &parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
tr.Run(nil)
if out.err != nil {
t.Fatal(out.err)
}
if !out.closed {
t.Fatalf("output RowReceiver not closed")
}
if result := out.rows.String(); result != c.expected {
t.Errorf("invalid results: %s, expected %s'", result, c.expected)
}
}
}
func dIntFnOrNull(fn func() (int32, bool)) parser.Datum {
if n, ok := fn(); ok {
return parser.NewDInt(parser.DInt(n))
}
return parser.DNull
}
func TestEncDatum(t *testing.T) {
a := &DatumAlloc{}
x := &EncDatum{}
if !x.IsUnset() {
t.Errorf("empty EncDatum should be unset")
}
if _, ok := x.Encoding(); ok {
t.Errorf("empty EncDatum has an encoding")
}
x.SetDatum(ColumnType_INT, parser.NewDInt(5))
if x.IsUnset() {
t.Errorf("unset after SetDatum()")
}
encoded, err := x.Encode(a, DatumEncoding_ASCENDING_KEY, nil)
if err != nil {
t.Fatal(err)
}
y := &EncDatum{}
y.SetEncoded(ColumnType_INT, DatumEncoding_ASCENDING_KEY, encoded)
if y.IsUnset() {
t.Errorf("unset after SetEncoded()")
}
if enc, ok := y.Encoding(); !ok {
t.Error("no encoding after SetEncoded")
} else if enc != DatumEncoding_ASCENDING_KEY {
t.Errorf("invalid encoding %d", enc)
}
err = y.Decode(a)
if err != nil {
t.Fatal(err)
}
if cmp := y.Datum.Compare(x.Datum); cmp != 0 {
t.Errorf("Datums should be equal, cmp = %d", cmp)
}
enc2, err := y.Encode(a, DatumEncoding_DESCENDING_KEY, nil)
if err != nil {
t.Fatal(err)
}
// y's encoding should not change.
if enc, ok := y.Encoding(); !ok {
t.Error("no encoding")
} else if enc != DatumEncoding_ASCENDING_KEY {
t.Errorf("invalid encoding %d", enc)
}
x.SetEncoded(ColumnType_INT, DatumEncoding_DESCENDING_KEY, enc2)
if enc, ok := x.Encoding(); !ok {
t.Error("no encoding")
} else if enc != DatumEncoding_DESCENDING_KEY {
t.Errorf("invalid encoding %d", enc)
}
err = x.Decode(a)
if err != nil {
t.Fatal(err)
}
if cmp := y.Datum.Compare(x.Datum); cmp != 0 {
t.Errorf("Datums should be equal, cmp = %d", cmp)
}
}
func TestUnorderedSync(t *testing.T) {
mrc := &MultiplexedRowChannel{}
mrc.Init(5)
for i := 1; i <= 5; i++ {
go func(i int) {
for j := 1; j <= 100; j++ {
var a, b sqlbase.EncDatum
a.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(i)))
b.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(j)))
row := sqlbase.EncDatumRow{a, b}
mrc.PushRow(row)
}
mrc.Close(nil)
}(i)
}
var retRows sqlbase.EncDatumRows
for {
row, err := mrc.NextRow()
if err != nil {
t.Fatal(err)
}
if row == nil {
break
}
retRows = append(retRows, row)
}
// Verify all elements.
for i := 1; i <= 5; i++ {
j := 1
for _, row := range retRows {
if int(*(row[0].Datum.(*parser.DInt))) == i {
if int(*(row[1].Datum.(*parser.DInt))) != j {
t.Errorf("Expected [%d %d], got %s", i, j, row)
}
j++
}
}
if j != 101 {
t.Errorf("Missing [%d %d]", i, j)
}
}
// Test case when one source closes with an error.
mrc = &MultiplexedRowChannel{}
mrc.Init(5)
for i := 1; i <= 5; i++ {
go func(i int) {
for j := 1; j <= 100; j++ {
var a, b sqlbase.EncDatum
a.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(i)))
b.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(j)))
row := sqlbase.EncDatumRow{a, b}
mrc.PushRow(row)
}
var err error
if i == 3 {
err = fmt.Errorf("Test error")
}
mrc.Close(err)
}(i)
}
for {
row, err := mrc.NextRow()
if err != nil {
if err.Error() != "Test error" {
t.Error(err)
}
break
}
if row == nil {
t.Error("Did not receive expected error")
}
}
}
func TestJoinReader(t *testing.T) {
defer leaktest.AfterTest(t)()
s, sqlDB, kvDB := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
// Create a table where each row is:
//
// | a | b | sum | s |
// |-----------------------------------------------------------------|
// | rowId/10 | rowId%10 | rowId/10 + rowId%10 | IntToEnglish(rowId) |
aFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row / 10))
}
bFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row % 10))
}
sumFn := func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row/10 + row%10))
}
sqlutils.CreateTable(t, sqlDB, "t",
"a INT, b INT, sum INT, s STRING, PRIMARY KEY (a,b), INDEX bs (b,s)",
99,
sqlutils.ToRowFn(aFn, bFn, sumFn, sqlutils.RowEnglishFn))
td := sqlbase.GetTableDescriptor(kvDB, "test", "t")
testCases := []struct {
spec JoinReaderSpec
input [][]parser.Datum
expected string
}{
{
spec: JoinReaderSpec{
OutputColumns: []uint32{0, 1, 2},
},
input: [][]parser.Datum{
{aFn(2), bFn(2)},
{aFn(5), bFn(5)},
{aFn(10), bFn(10)},
{aFn(15), bFn(15)},
},
expected: "[[0 2 2] [0 5 5] [1 0 1] [1 5 6]]",
},
{
spec: JoinReaderSpec{
Filter: Expression{Expr: "$2 <= 5"}, // sum <= 5
OutputColumns: []uint32{3},
},
input: [][]parser.Datum{
{aFn(1), bFn(1)},
{aFn(25), bFn(25)},
{aFn(5), bFn(5)},
{aFn(21), bFn(21)},
{aFn(34), bFn(34)},
{aFn(13), bFn(13)},
{aFn(51), bFn(51)},
{aFn(50), bFn(50)},
},
expected: "[['one'] ['five'] ['two-one'] ['one-three'] ['five-zero']]",
},
}
for _, c := range testCases {
js := c.spec
js.Table = *td
txn := client.NewTxn(context.Background(), *kvDB)
in := &RowBuffer{}
for _, row := range c.input {
encRow := make(sqlbase.EncDatumRow, len(row))
for i, d := range row {
encRow[i].SetDatum(sqlbase.ColumnType_INT, d)
}
in.rows = append(in.rows, encRow)
}
out := &RowBuffer{}
jr, err := newJoinReader(&js, txn, in, out, &parser.EvalContext{})
if err != nil {
t.Fatal(err)
}
jr.Run(nil)
if out.err != nil {
t.Fatal(out.err)
}
if !in.done {
t.Fatal("joinReader stopped accepting rows")
}
if !out.closed {
t.Fatalf("output RowReceiver not closed")
}
if result := out.rows.String(); result != c.expected {
t.Errorf("invalid results: %s, expected %s'", result, c.expected)
}
}
}
func TestAdminAPITableDetailsZone(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
// Create database and table.
session := sql.NewSession(sql.SessionArgs{User: security.RootUser}, ts.sqlExecutor, nil)
setupQueries := []string{
"CREATE DATABASE test",
"CREATE TABLE test.tbl (val STRING)",
}
for _, q := range setupQueries {
res := ts.sqlExecutor.ExecuteStatements(context.Background(), session, q, nil)
if res.ResultList[0].Err != nil {
t.Fatalf("error executing '%s': %s", q, res.ResultList[0].Err)
}
}
// Function to verify the zone for test.tbl as returned by the Admin API.
verifyZone := func(expectedZone config.ZoneConfig, expectedLevel serverpb.ZoneConfigurationLevel) {
var resp serverpb.TableDetailsResponse
if err := apiGet(s, "databases/test/tables/tbl", &resp); err != nil {
t.Fatal(err)
}
if a, e := &resp.ZoneConfig, &expectedZone; !proto.Equal(a, e) {
t.Errorf("actual zone config %v did not match expected value %v", a, e)
}
if a, e := resp.ZoneConfigLevel, expectedLevel; a != e {
t.Errorf("actual ZoneConfigurationLevel %s did not match expected value %s", a, e)
}
if t.Failed() {
t.FailNow()
}
}
// Function to store a zone config for a given object ID.
setZone := func(zoneCfg config.ZoneConfig, id sqlbase.ID) {
zoneBytes, err := zoneCfg.Marshal()
if err != nil {
t.Fatal(err)
}
const query = `INSERT INTO system.zones VALUES($1, $2)`
params := parser.NewPlaceholderInfo()
params.SetValue(`1`, parser.NewDInt(parser.DInt(id)))
params.SetValue(`2`, parser.NewDBytes(parser.DBytes(zoneBytes)))
res := ts.sqlExecutor.ExecuteStatements(context.Background(), session, query, params)
if res.ResultList[0].Err != nil {
t.Fatalf("error executing '%s': %s", query, res.ResultList[0].Err)
}
}
// Verify zone matches cluster default.
verifyZone(config.DefaultZoneConfig(), serverpb.ZoneConfigurationLevel_CLUSTER)
// Get ID path for table. This will be an array of three IDs, containing the ID of the root namespace,
// the database, and the table (in that order).
idPath, err := ts.admin.queryDescriptorIDPath(context.Background(), session, []string{"test", "tbl"})
if err != nil {
t.Fatal(err)
}
// Apply zone configuration to database and check again.
dbZone := config.ZoneConfig{
RangeMinBytes: 456,
}
setZone(dbZone, idPath[1])
verifyZone(dbZone, serverpb.ZoneConfigurationLevel_DATABASE)
// Apply zone configuration to table and check again.
tblZone := config.ZoneConfig{
RangeMinBytes: 789,
}
setZone(tblZone, idPath[2])
verifyZone(tblZone, serverpb.ZoneConfigurationLevel_TABLE)
}
func TestValues(t *testing.T) {
defer leaktest.AfterTest(t)()
p := makePlanner()
vInt := int64(5)
vNum := 3.14159
vStr := "two furs one cub"
vBool := true
unsupp := &parser.RangeCond{}
intVal := func(v int64) *parser.NumVal {
return &parser.NumVal{Value: constant.MakeInt64(v)}
}
floatVal := func(f float64) *parser.CastExpr {
return &parser.CastExpr{
Expr: &parser.NumVal{Value: constant.MakeFloat64(f)},
Type: &parser.FloatColType{},
}
}
asRow := func(datums ...parser.Datum) []parser.DTuple {
return []parser.DTuple{datums}
}
makeValues := func(tuples ...*parser.Tuple) *parser.ValuesClause {
return &parser.ValuesClause{Tuples: tuples}
}
makeTuple := func(exprs ...parser.Expr) *parser.Tuple {
return &parser.Tuple{Exprs: exprs}
}
testCases := []struct {
stmt *parser.ValuesClause
rows []parser.DTuple
ok bool
}{
{
makeValues(makeTuple(intVal(vInt))),
asRow(parser.NewDInt(parser.DInt(vInt))),
true,
},
{
makeValues(makeTuple(intVal(vInt), intVal(vInt))),
asRow(parser.NewDInt(parser.DInt(vInt)), parser.NewDInt(parser.DInt(vInt))),
true,
},
{
makeValues(makeTuple(floatVal(vNum))),
asRow(parser.NewDFloat(parser.DFloat(vNum))),
true,
},
{
makeValues(makeTuple(parser.NewDString(vStr))),
asRow(parser.NewDString(vStr)),
true,
},
{
makeValues(makeTuple(parser.NewDBytes(parser.DBytes(vStr)))),
asRow(parser.NewDBytes(parser.DBytes(vStr))),
true,
},
{
makeValues(makeTuple(parser.MakeDBool(parser.DBool(vBool)))),
asRow(parser.MakeDBool(parser.DBool(vBool))),
true,
},
{
makeValues(makeTuple(unsupp)),
nil,
false,
},
}
for i, tc := range testCases {
plan, err := func() (_ planNode, err error) {
defer func() {
if r := recover(); r != nil {
err = errors.Errorf("%v", r)
}
}()
return p.ValuesClause(tc.stmt, nil)
}()
if err == nil != tc.ok {
t.Errorf("%d: error_expected=%t, but got error %v", i, tc.ok, err)
}
if plan != nil {
if err := plan.expandPlan(); err != nil {
t.Errorf("%d: unexpected error in expandPlan: %v", i, err)
continue
}
if err := plan.Start(); err != nil {
t.Errorf("%d: unexpected error in Start: %v", i, err)
continue
}
var rows []parser.DTuple
next, err := plan.Next()
for ; next; next, err = plan.Next() {
rows = append(rows, plan.Values())
}
if err != nil {
t.Error(err)
continue
}
if !reflect.DeepEqual(rows, tc.rows) {
t.Errorf("%d: expected rows:\n%+v\nactual rows:\n%+v", i, tc.rows, rows)
}
}
}
}
func dumpTable(w io.Writer, conn *sqlConn, origDBName, origTableName string) error {
const limit = 100
// Escape names since they can't be used in placeholders.
dbname := parser.Name(origDBName).String()
tablename := parser.Name(origTableName).String()
if err := conn.Exec(fmt.Sprintf("SET DATABASE = %s", dbname), nil); err != nil {
return err
}
// Fetch all table metadata in a transaction and its time to guarantee it
// doesn't change between the various SHOW statements.
if err := conn.Exec("BEGIN", nil); err != nil {
return err
}
vals, err := conn.QueryRow("SELECT cluster_logical_timestamp()::int", nil)
if err != nil {
return err
}
clusterTSStart := vals[0].(int64)
clusterTS := time.Unix(0, clusterTSStart).Format(time.RFC3339Nano)
// A previous version of the code did a SELECT on system.descriptor. This
// required the SELECT privilege to the descriptor table, which only root
// has. Allowing non-root to do this would let users see other users' table
// descriptors which is a problem in multi-tenancy.
// Fetch column types.
rows, err := conn.Query(fmt.Sprintf("SHOW COLUMNS FROM %s", tablename), nil)
if err != nil {
return err
}
vals = make([]driver.Value, 2)
coltypes := make(map[string]string)
for {
if err := rows.Next(vals); err == io.EOF {
break
} else if err != nil {
return err
}
name, ok := vals[0].(string)
if !ok {
return fmt.Errorf("unexpected value: %T", vals[1])
}
typ, ok := vals[1].(string)
if !ok {
return fmt.Errorf("unexpected value: %T", vals[4])
}
coltypes[name] = typ
}
if err := rows.Close(); err != nil {
return err
}
// index holds the names, in order, of the primary key columns.
var index []string
// Primary index is always the first index returned by SHOW INDEX.
rows, err = conn.Query(fmt.Sprintf("SHOW INDEX FROM %s", tablename), nil)
if err != nil {
return err
}
vals = make([]driver.Value, 5)
var primaryIndex string
// Find the primary index columns.
for {
if err := rows.Next(vals); err == io.EOF {
break
} else if err != nil {
return err
}
b, ok := vals[1].(string)
if !ok {
return fmt.Errorf("unexpected value: %T", vals[1])
}
if primaryIndex == "" {
primaryIndex = b
} else if primaryIndex != b {
break
}
b, ok = vals[4].(string)
if !ok {
return fmt.Errorf("unexpected value: %T", vals[4])
}
index = append(index, parser.Name(b).String())
}
if err := rows.Close(); err != nil {
return err
}
if len(index) == 0 {
return fmt.Errorf("no primary key index found")
}
indexes := strings.Join(index, ", ")
// Build the SELECT query.
var sbuf bytes.Buffer
fmt.Fprintf(&sbuf, "SELECT %s, * FROM %s@%s AS OF SYSTEM TIME '%s'", indexes, tablename, primaryIndex, clusterTS)
var wbuf bytes.Buffer
fmt.Fprintf(&wbuf, " WHERE ROW (%s) > ROW (", indexes)
for i := range index {
if i > 0 {
wbuf.WriteString(", ")
}
fmt.Fprintf(&wbuf, "$%d", i+1)
}
wbuf.WriteString(")")
// No WHERE clause first time, so add a place to inject it.
fmt.Fprintf(&sbuf, "%%s ORDER BY %s LIMIT %d", indexes, limit)
bs := sbuf.String()
vals, err = conn.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", tablename), nil)
if err != nil {
return err
}
create := vals[1].(string)
if _, err := w.Write([]byte(create)); err != nil {
return err
}
if _, err := w.Write([]byte(";\n")); err != nil {
return err
}
if err := conn.Exec("COMMIT", nil); err != nil {
return err
}
// pk holds the last values of the fetched primary keys
var pk []driver.Value
q := fmt.Sprintf(bs, "")
for {
rows, err := conn.Query(q, pk)
if err != nil {
return err
}
cols := rows.Columns()
pkcols := cols[:len(index)]
cols = cols[len(index):]
inserts := make([][]string, 0, limit)
i := 0
for i < limit {
vals := make([]driver.Value, len(cols)+len(pkcols))
if err := rows.Next(vals); err == io.EOF {
break
} else if err != nil {
return err
}
if pk == nil {
q = fmt.Sprintf(bs, wbuf.String())
}
pk = vals[:len(index)]
vals = vals[len(index):]
ivals := make([]string, len(vals))
// Values need to be correctly encoded for INSERT statements in a text file.
for si, sv := range vals {
switch t := sv.(type) {
case nil:
ivals[si] = "NULL"
case bool:
ivals[si] = parser.MakeDBool(parser.DBool(t)).String()
case int64:
ivals[si] = parser.NewDInt(parser.DInt(t)).String()
case float64:
ivals[si] = parser.NewDFloat(parser.DFloat(t)).String()
case string:
ivals[si] = parser.NewDString(t).String()
case []byte:
switch ct := coltypes[cols[si]]; ct {
case "INTERVAL":
ivals[si] = fmt.Sprintf("'%s'", t)
case "BYTES":
ivals[si] = parser.NewDBytes(parser.DBytes(t)).String()
default:
// STRING and DECIMAL types can have optional length
// suffixes, so only examine the prefix of the type.
if strings.HasPrefix(coltypes[cols[si]], "STRING") {
ivals[si] = parser.NewDString(string(t)).String()
} else if strings.HasPrefix(coltypes[cols[si]], "DECIMAL") {
ivals[si] = string(t)
} else {
panic(errors.Errorf("unknown []byte type: %s, %v: %s", t, cols[si], coltypes[cols[si]]))
}
}
case time.Time:
var d parser.Datum
ct := coltypes[cols[si]]
switch ct {
case "DATE":
d = parser.NewDDateFromTime(t, time.UTC)
case "TIMESTAMP":
d = parser.MakeDTimestamp(t, time.Nanosecond)
case "TIMESTAMP WITH TIME ZONE":
d = parser.MakeDTimestampTZ(t, time.Nanosecond)
default:
panic(errors.Errorf("unknown timestamp type: %s, %v: %s", t, cols[si], coltypes[cols[si]]))
}
ivals[si] = fmt.Sprintf("'%s'", d)
default:
panic(errors.Errorf("unknown field type: %T (%s)", t, cols[si]))
}
}
inserts = append(inserts, ivals)
i++
}
for si, sv := range pk {
b, ok := sv.([]byte)
if ok && strings.HasPrefix(coltypes[pkcols[si]], "STRING") {
// Primary key strings need to be converted to a go string, but not SQL
// encoded since they aren't being written to a text file.
pk[si] = string(b)
}
}
if err := rows.Close(); err != nil {
return err
}
if i == 0 {
break
}
fmt.Fprintf(w, "\nINSERT INTO %s VALUES", tablename)
for idx, values := range inserts {
if idx > 0 {
fmt.Fprint(w, ",")
}
fmt.Fprint(w, "\n\t(")
for vi, v := range values {
if vi > 0 {
fmt.Fprint(w, ", ")
}
fmt.Fprint(w, v)
}
fmt.Fprint(w, ")")
}
fmt.Fprintln(w, ";")
if i < limit {
break
}
}
return nil
}
populate: func(p *planner, addRow func(...parser.Datum)) error {
return forEachTableDesc(p,
func(db *sqlbase.DatabaseDescriptor, table *sqlbase.TableDescriptor) {
// Table descriptors already holds columns in-order.
visible := 0
for _, column := range table.Columns {
if column.Hidden {
continue
}
visible++
addRow(
defString, // table_catalog
parser.NewDString(db.Name), // table_schema
parser.NewDString(table.Name), // table_name
parser.NewDString(column.Name), // column_name
parser.NewDInt(parser.DInt(visible)), // ordinal_position, 1-indexed
dStringOrNull(column.DefaultExpr), // column_default
yesOrNoDatum(column.Nullable), // is_nullable
parser.NewDString(column.Type.Kind.String()), // data_type
characterMaximumLength(column.Type), // character_maximum_length
characterOctetLength(column.Type), // character_octet_length
numericPrecision(column.Type), // numeric_precision
numericScale(column.Type), // numeric_scale
datetimePrecision(column.Type), // datetime_precision
)
}
},
)
},
}
func TestClusterFlow(t *testing.T) {
defer leaktest.AfterTest(t)()
const numRows = 100
args := base.TestClusterArgs{ReplicationMode: base.ReplicationManual}
tc := serverutils.StartTestCluster(t, 3, args)
defer tc.Stopper().Stop()
sumDigitsFn := func(row int) parser.Datum {
sum := 0
for row > 0 {
sum += row % 10
row /= 10
}
return parser.NewDInt(parser.DInt(sum))
}
sqlutils.CreateTable(t, tc.ServerConn(0), "t",
"num INT PRIMARY KEY, digitsum INT, numstr STRING, INDEX s (digitsum)",
numRows,
sqlutils.ToRowFn(sqlutils.RowIdxFn, sumDigitsFn, sqlutils.RowEnglishFn))
kvDB := tc.Server(0).KVClient().(*client.DB)
desc := sqlbase.GetTableDescriptor(kvDB, "test", "t")
makeIndexSpan := func(start, end int) TableReaderSpan {
var span roachpb.Span
prefix := roachpb.Key(sqlbase.MakeIndexKeyPrefix(desc, desc.Indexes[0].ID))
span.Key = append(prefix, encoding.EncodeVarintAscending(nil, int64(start))...)
span.EndKey = append(span.EndKey, prefix...)
span.EndKey = append(span.EndKey, encoding.EncodeVarintAscending(nil, int64(end))...)
return TableReaderSpan{Span: span}
}
// Set up table readers on three hosts feeding data into a join reader on
// the third host. This is a basic test for the distributed flow
// infrastructure, including local and remote streams.
//
// Note that the ranges won't necessarily be local to the table readers, but
// that doesn't matter for the purposes of this test.
tr1 := TableReaderSpec{
Table: *desc,
IndexIdx: 1,
OutputColumns: []uint32{0, 1},
Spans: []TableReaderSpan{makeIndexSpan(0, 8)},
}
tr2 := TableReaderSpec{
Table: *desc,
IndexIdx: 1,
OutputColumns: []uint32{0, 1},
Spans: []TableReaderSpan{makeIndexSpan(8, 12)},
}
tr3 := TableReaderSpec{
Table: *desc,
IndexIdx: 1,
OutputColumns: []uint32{0, 1},
Spans: []TableReaderSpan{makeIndexSpan(12, 100)},
}
jr := JoinReaderSpec{
Table: *desc,
OutputColumns: []uint32{2},
}
txn := client.NewTxn(context.Background(), *kvDB)
fid := FlowID{uuid.MakeV4()}
req1 := &SetupFlowRequest{Txn: txn.Proto}
req1.Flow = FlowSpec{
FlowID: fid,
Processors: []ProcessorSpec{{
Core: ProcessorCoreUnion{TableReader: &tr1},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{
{Mailbox: &MailboxSpec{StreamID: 0, TargetAddr: tc.Server(2).ServingAddr()}},
},
}},
}},
}
req2 := &SetupFlowRequest{Txn: txn.Proto}
req2.Flow = FlowSpec{
FlowID: fid,
Processors: []ProcessorSpec{{
Core: ProcessorCoreUnion{TableReader: &tr2},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{
{Mailbox: &MailboxSpec{StreamID: 1, TargetAddr: tc.Server(2).ServingAddr()}},
},
}},
}},
}
req3 := &SetupFlowRequest{Txn: txn.Proto}
req3.Flow = FlowSpec{
FlowID: fid,
Processors: []ProcessorSpec{
{
Core: ProcessorCoreUnion{TableReader: &tr3},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{
{LocalStreamID: LocalStreamID(0)},
},
}},
},
{
Input: []InputSyncSpec{{
Type: InputSyncSpec_ORDERED,
Ordering: Ordering{Columns: []Ordering_Column{{1, Ordering_Column_ASC}}},
Streams: []StreamEndpointSpec{
{Mailbox: &MailboxSpec{StreamID: 0}},
{Mailbox: &MailboxSpec{StreamID: 1}},
{LocalStreamID: LocalStreamID(0)},
},
}},
Core: ProcessorCoreUnion{JoinReader: &jr},
Output: []OutputRouterSpec{{
Type: OutputRouterSpec_MIRROR,
Streams: []StreamEndpointSpec{{Mailbox: &MailboxSpec{SimpleResponse: true}}},
}}},
},
}
var clients []DistSQLClient
for i := 0; i < 3; i++ {
s := tc.Server(i)
conn, err := s.RPCContext().GRPCDial(s.ServingAddr())
if err != nil {
t.Fatal(err)
}
clients = append(clients, NewDistSQLClient(conn))
}
ctx := context.Background()
if log.V(1) {
log.Infof(ctx, "Setting up flow on 0")
}
if resp, err := clients[0].SetupFlow(context.Background(), req1); err != nil {
t.Fatal(err)
} else if resp.Error != nil {
t.Fatal(resp.Error)
}
if log.V(1) {
log.Infof(ctx, "Setting up flow on 1")
}
if resp, err := clients[1].SetupFlow(context.Background(), req2); err != nil {
t.Fatal(err)
} else if resp.Error != nil {
t.Fatal(resp.Error)
}
if log.V(1) {
log.Infof(ctx, "Running flow on 2")
}
stream, err := clients[2].RunSimpleFlow(context.Background(), req3)
if err != nil {
t.Fatal(err)
}
var decoder StreamDecoder
var rows sqlbase.EncDatumRows
for {
msg, err := stream.Recv()
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
err = decoder.AddMessage(msg)
if err != nil {
t.Fatal(err)
}
rows = testGetDecodedRows(t, &decoder, rows)
}
if done, trailerErr := decoder.IsDone(); !done {
t.Fatal("stream not done")
} else if trailerErr != nil {
t.Fatal("error in the stream trailer:", trailerErr)
}
// The result should be all the numbers in string form, ordered by the
// digit sum (and then by number).
var results []string
for sum := 1; sum <= 50; sum++ {
for i := 1; i <= numRows; i++ {
if int(*sumDigitsFn(i).(*parser.DInt)) == sum {
results = append(results, fmt.Sprintf("['%s']", sqlutils.IntToEnglish(i)))
}
}
}
expected := strings.Join(results, " ")
expected = "[" + expected + "]"
if rowStr := rows.String(); rowStr != expected {
t.Errorf("Result: %s\n Expected: %s\n", rowStr, expected)
}
}
func TestOrderedSync(t *testing.T) {
v := [6]sqlbase.EncDatum{}
for i := range v {
v[i].SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(i)))
}
asc := encoding.Ascending
desc := encoding.Descending
testCases := []struct {
sources []sqlbase.EncDatumRows
ordering sqlbase.ColumnOrdering
expected sqlbase.EncDatumRows
}{
{
sources: []sqlbase.EncDatumRows{
{
{v[0], v[1], v[4]},
{v[0], v[1], v[2]},
{v[0], v[2], v[3]},
{v[1], v[1], v[3]},
},
{
{v[1], v[0], v[4]},
},
{
{v[0], v[0], v[0]},
{v[4], v[4], v[4]},
},
},
ordering: sqlbase.ColumnOrdering{
{ColIdx: 0, Direction: asc},
{ColIdx: 1, Direction: asc},
},
expected: sqlbase.EncDatumRows{
{v[0], v[0], v[0]},
{v[0], v[1], v[4]},
{v[0], v[1], v[2]},
{v[0], v[2], v[3]},
{v[1], v[0], v[4]},
{v[1], v[1], v[3]},
{v[4], v[4], v[4]},
},
},
{
sources: []sqlbase.EncDatumRows{
{},
{
{v[1], v[0], v[4]},
},
{
{v[3], v[4], v[1]},
{v[4], v[4], v[4]},
{v[3], v[2], v[0]},
},
{
{v[4], v[4], v[5]},
{v[3], v[3], v[0]},
{v[0], v[0], v[0]},
},
},
ordering: sqlbase.ColumnOrdering{
{ColIdx: 1, Direction: desc},
{ColIdx: 0, Direction: asc},
{ColIdx: 2, Direction: asc},
},
expected: sqlbase.EncDatumRows{
{v[3], v[4], v[1]},
{v[4], v[4], v[4]},
{v[4], v[4], v[5]},
{v[3], v[3], v[0]},
{v[3], v[2], v[0]},
{v[0], v[0], v[0]},
{v[1], v[0], v[4]},
},
},
}
for testIdx, c := range testCases {
var sources []RowSource
for _, srcRows := range c.sources {
rowBuf := &RowBuffer{rows: srcRows}
sources = append(sources, rowBuf)
}
src, err := makeOrderedSync(c.ordering, sources)
if err != nil {
t.Fatal(err)
}
var retRows sqlbase.EncDatumRows
for {
row, err := src.NextRow()
if err != nil {
t.Fatal(err)
}
if row == nil {
break
}
retRows = append(retRows, row)
}
expStr := c.expected.String()
retStr := retRows.String()
if expStr != retStr {
t.Errorf("invalid results for case %d; expected:\n %s\ngot:\n %s",
testIdx, expStr, retStr)
}
}
}
// RowIdxFn is a GenValueFn that returns the row number as a DInt
func RowIdxFn(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row))
}
// RowModuloFn creates a GenValueFn that returns the row number modulo a given
// value as a DInt
func RowModuloFn(modulo int) GenValueFn {
return func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row % modulo))
}
}
func TestEncDatumRowCompare(t *testing.T) {
v := [5]EncDatum{}
for i := range v {
v[i].SetDatum(ColumnType_INT, parser.NewDInt(parser.DInt(i)))
}
asc := encoding.Ascending
desc := encoding.Descending
testCases := []struct {
row1, row2 EncDatumRow
ord ColumnOrdering
cmp int
}{
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{},
cmp: 0,
},
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{{1, desc}},
cmp: 0,
},
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{{0, asc}, {1, desc}},
cmp: 0,
},
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{{2, asc}},
cmp: -1,
},
{
row1: EncDatumRow{v[0], v[1], v[3]},
row2: EncDatumRow{v[0], v[1], v[2]},
ord: ColumnOrdering{{2, asc}},
cmp: 1,
},
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{{2, asc}, {0, asc}, {1, asc}},
cmp: -1,
},
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{{0, asc}, {2, desc}},
cmp: 1,
},
{
row1: EncDatumRow{v[0], v[1], v[2]},
row2: EncDatumRow{v[0], v[1], v[3]},
ord: ColumnOrdering{{1, desc}, {0, asc}, {2, desc}},
cmp: 1,
},
{
row1: EncDatumRow{v[2], v[3]},
row2: EncDatumRow{v[1], v[3], v[0]},
ord: ColumnOrdering{{0, asc}},
cmp: 1,
},
{
row1: EncDatumRow{v[2], v[3]},
row2: EncDatumRow{v[1], v[3], v[0]},
ord: ColumnOrdering{{1, desc}, {0, asc}},
cmp: 1,
},
{
row1: EncDatumRow{v[2], v[3]},
row2: EncDatumRow{v[1], v[3], v[0]},
ord: ColumnOrdering{{1, asc}, {0, asc}},
cmp: 1,
},
{
row1: EncDatumRow{v[2], v[3]},
row2: EncDatumRow{v[1], v[3], v[0]},
ord: ColumnOrdering{{1, asc}, {0, desc}},
cmp: -1,
},
{
row1: EncDatumRow{v[2], v[3]},
row2: EncDatumRow{v[1], v[3], v[0]},
ord: ColumnOrdering{{0, desc}, {1, asc}},
cmp: -1,
},
}
a := &DatumAlloc{}
for _, c := range testCases {
cmp, err := c.row1.Compare(a, c.ord, c.row2)
if err != nil {
t.Error(err)
} else if cmp != c.cmp {
t.Errorf("%s cmp %s ordering %v got %d, expected %d",
c.row1, c.row2, c.ord, cmp, c.cmp)
}
}
}