func checkEquivExpr(a, b parser.TypedExpr, qvals qvalMap) error { // The expressions above only use the values 1 and 2. Verify that the // simplified expressions evaluate to the same value as the original // expression for interesting values. for _, v := range []parser.Datum{ parser.NewDInt(0), parser.NewDInt(1), parser.NewDInt(2), parser.NewDInt(3), parser.DNull, } { for _, q := range qvals { q.datum = v } da, err := a.Eval(parser.EvalContext{}) if err != nil { return fmt.Errorf("%s: %v", a, err) } db, err := b.Eval(parser.EvalContext{}) if err != nil { return fmt.Errorf("%s: %v", b, err) } // This is tricky: we don't require the expressions to produce identical // results, but to either both return true or both return not true (either // false or NULL). if (da == parser.DBoolTrue) != (db == parser.DBoolTrue) { return fmt.Errorf("%s: %s: expected %s, but found %s", a, v, da, db) } } return nil }
// golangFillQueryArguments populates the placeholder map with // types and values from an array of Go values. // TODO: This does not support arguments of the SQL 'Date' type, as there is not // an equivalent type in Go's standard library. It's not currently needed by any // of our internal tables. func golangFillQueryArguments(pinfo *parser.PlaceholderInfo, args []interface{}) { pinfo.Clear() for i, arg := range args { k := fmt.Sprint(i + 1) if arg == nil { pinfo.SetValue(k, parser.DNull) continue } // A type switch to handle a few explicit types with special semantics: // - Datums are passed along as is. // - Time datatypes get special representation in the database. var d parser.Datum switch t := arg.(type) { case parser.Datum: d = t case time.Time: d = parser.MakeDTimestamp(t, time.Microsecond) case time.Duration: d = &parser.DInterval{Duration: duration.Duration{Nanos: t.Nanoseconds()}} case *inf.Dec: dd := &parser.DDecimal{} dd.Set(t) d = dd } if d == nil { // Handle all types which have an underlying type that can be stored in the // database. // Note: if this reflection becomes a performance concern in the future, // commonly used types could be added explicitly into the type switch above // for a performance gain. val := reflect.ValueOf(arg) switch val.Kind() { case reflect.Bool: d = parser.MakeDBool(parser.DBool(val.Bool())) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: d = parser.NewDInt(parser.DInt(val.Int())) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: d = parser.NewDInt(parser.DInt(val.Uint())) case reflect.Float32, reflect.Float64: d = parser.NewDFloat(parser.DFloat(val.Float())) case reflect.String: d = parser.NewDString(val.String()) case reflect.Slice: // Handle byte slices. if val.Type().Elem().Kind() == reflect.Uint8 { d = parser.NewDBytes(parser.DBytes(val.Bytes())) } } if d == nil { panic(fmt.Sprintf("unexpected type %T", arg)) } } pinfo.SetValue(k, d) } }
// Arg implements the parser.Args interface. // TODO: This does not support arguments of the SQL 'Date' type, as there is not // an equivalent type in Go's standard library. It's not currently needed by any // of our internal tables. func (gp golangParameters) Arg(name string) (parser.Datum, bool) { i, err := processPositionalArgument(name) if err != nil { return nil, false } if i < 1 || int(i) > len(gp) { return nil, false } arg := gp[i-1] if arg == nil { return parser.DNull, true } // A type switch to handle a few explicit types with special semantics. switch t := arg.(type) { // Datums are passed along as is. case parser.Datum: return t, true // Time datatypes get special representation in the database. case time.Time: return parser.MakeDTimestamp(t, time.Microsecond), true case time.Duration: return &parser.DInterval{Duration: duration.Duration{Nanos: t.Nanoseconds()}}, true case *inf.Dec: dd := &parser.DDecimal{} dd.Set(t) return dd, true } // Handle all types which have an underlying type that can be stored in the // database. // Note: if this reflection becomes a performance concern in the future, // commonly used types could be added explicitly into the type switch above // for a performance gain. val := reflect.ValueOf(arg) switch val.Kind() { case reflect.Bool: return parser.MakeDBool(parser.DBool(val.Bool())), true case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return parser.NewDInt(parser.DInt(val.Int())), true case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: return parser.NewDInt(parser.DInt(val.Uint())), true case reflect.Float32, reflect.Float64: return parser.NewDFloat(parser.DFloat(val.Float())), true case reflect.String: return parser.NewDString(val.String()), true case reflect.Slice: // Handle byte slices. if val.Type().Elem().Kind() == reflect.Uint8 { return parser.NewDBytes(parser.DBytes(val.Bytes())), true } } panic(fmt.Sprintf("unexpected type %T", arg)) }
// Events is an endpoint that returns the latest event log entries, with the following // optional URL parameters: // // type=STRING returns events with this type (e.g. "create_table") // targetID=INT returns events for that have this targetID func (s *adminServer) Events(ctx context.Context, req *serverpb.EventsRequest) (*serverpb.EventsResponse, error) { args := sql.SessionArgs{User: s.getUser(req)} session := sql.NewSession(ctx, args, s.server.sqlExecutor, nil) // Execute the query. q := makeSQLQuery() q.Append("SELECT timestamp, eventType, targetID, reportingID, info, uniqueID ") q.Append("FROM system.eventlog ") q.Append("WHERE true ") // This simplifies the WHERE clause logic below. if len(req.Type) > 0 { q.Append("AND eventType = $ ", parser.NewDString(req.Type)) } if req.TargetId > 0 { q.Append("AND targetID = $ ", parser.NewDInt(parser.DInt(req.TargetId))) } q.Append("ORDER BY timestamp DESC ") q.Append("LIMIT $", parser.NewDInt(parser.DInt(apiEventLimit))) if len(q.Errors()) > 0 { return nil, s.serverErrors(q.Errors()) } r := s.server.sqlExecutor.ExecuteStatements(session, q.String(), q.QueryArguments()) if err := s.checkQueryResults(r.ResultList, 1); err != nil { return nil, s.serverError(err) } // Marshal response. var resp serverpb.EventsResponse scanner := makeResultScanner(r.ResultList[0].Columns) for _, row := range r.ResultList[0].Rows { var event serverpb.EventsResponse_Event var ts time.Time if err := scanner.ScanIndex(row, 0, &ts); err != nil { return nil, err } event.Timestamp = serverpb.EventsResponse_Event_Timestamp{Sec: ts.Unix(), Nsec: uint32(ts.Nanosecond())} if err := scanner.ScanIndex(row, 1, &event.EventType); err != nil { return nil, err } if err := scanner.ScanIndex(row, 2, &event.TargetID); err != nil { return nil, err } if err := scanner.ScanIndex(row, 3, &event.ReportingID); err != nil { return nil, err } if err := scanner.ScanIndex(row, 4, &event.Info); err != nil { return nil, err } if err := scanner.ScanIndex(row, 5, &event.UniqueID); err != nil { return nil, err } resp.Events = append(resp.Events, event) } return &resp, nil }
// queryNamespaceID queries for the ID of the namespace with the given name and // parent ID. func (s *adminServer) queryNamespaceID( session *sql.Session, parentID sqlbase.ID, name string, ) (sqlbase.ID, error) { const query = `SELECT id FROM system.namespace WHERE parentID = $1 AND name = $2` params := parser.NewPlaceholderInfo() params.SetValue(`1`, parser.NewDInt(parser.DInt(parentID))) params.SetValue(`2`, parser.NewDString(name)) r := s.server.sqlExecutor.ExecuteStatements(session, query, params) if err := s.checkQueryResults(r.ResultList, 1); err != nil { return 0, err } result := r.ResultList[0] if len(result.Rows) == 0 { return 0, errors.Errorf("namespace %s with ParentID %d not found", name, parentID) } var id int64 scanner := resultScanner{} err := scanner.ScanIndex(result.Rows[0], 0, &id) if err != nil { return 0, err } return sqlbase.ID(id), nil }
// queryZone retrieves the specific ZoneConfig associated with the supplied ID, // if it exists. func (s *adminServer) queryZone( session *sql.Session, id sqlbase.ID, ) (config.ZoneConfig, bool, error) { const query = `SELECT config FROM system.zones WHERE id = $1` params := parser.NewPlaceholderInfo() params.SetValue(`1`, parser.NewDInt(parser.DInt(id))) r := s.server.sqlExecutor.ExecuteStatements(session, query, params) if err := s.checkQueryResults(r.ResultList, 1); err != nil { return config.ZoneConfig{}, false, err } result := r.ResultList[0] if len(result.Rows) == 0 { return config.ZoneConfig{}, false, nil } var zoneBytes []byte scanner := resultScanner{} err := scanner.ScanIndex(result.Rows[0], 0, &zoneBytes) if err != nil { return config.ZoneConfig{}, false, err } var zone config.ZoneConfig if err := zone.Unmarshal(zoneBytes); err != nil { return config.ZoneConfig{}, false, err } return zone, true, nil }
func makeIntTestDatum(count int) []parser.Datum { rng, _ := randutil.NewPseudoRand() vals := make([]parser.Datum, count) for i := range vals { vals[i] = parser.NewDInt(parser.DInt(rng.Int63())) } return vals }
// ShowIndex returns all the indexes for a table. // Privileges: Any privilege on table. // Notes: postgres does not have a SHOW INDEXES statement. // mysql requires some privilege for any column. func (p *planner) ShowIndex(n *parser.ShowIndex) (planNode, error) { tn, err := n.Table.NormalizeWithDatabaseName(p.session.Database) if err != nil { return nil, err } desc, err := p.mustGetTableDesc(tn) if err != nil { return nil, err } if err := p.anyPrivilege(desc); err != nil { return nil, err } v := &valuesNode{ columns: []ResultColumn{ {Name: "Table", Typ: parser.TypeString}, {Name: "Name", Typ: parser.TypeString}, {Name: "Unique", Typ: parser.TypeBool}, {Name: "Seq", Typ: parser.TypeInt}, {Name: "Column", Typ: parser.TypeString}, {Name: "Direction", Typ: parser.TypeString}, {Name: "Storing", Typ: parser.TypeBool}, }, } appendRow := func(index sqlbase.IndexDescriptor, colName string, sequence int, direction string, isStored bool) { v.rows = append(v.rows, []parser.Datum{ parser.NewDString(tn.Table()), parser.NewDString(index.Name), parser.MakeDBool(parser.DBool(index.Unique)), parser.NewDInt(parser.DInt(sequence)), parser.NewDString(colName), parser.NewDString(direction), parser.MakeDBool(parser.DBool(isStored)), }) } for _, index := range append([]sqlbase.IndexDescriptor{desc.PrimaryIndex}, desc.Indexes...) { sequence := 1 for i, col := range index.ColumnNames { appendRow(index, col, sequence, index.ColumnDirections[i].String(), false) sequence++ } for _, col := range index.StoreColumnNames { appendRow(index, col, sequence, "N/A", true) sequence++ } } return v, nil }
// RandDatum generates a random Datum of the given type. // If null is true, the datum can be DNull. func RandDatum(rng *rand.Rand, typ ColumnType_Kind, null bool) parser.Datum { if null && rng.Intn(10) == 0 { return parser.DNull } switch typ { case ColumnType_BOOL: return parser.MakeDBool(rng.Intn(2) == 1) case ColumnType_INT: return parser.NewDInt(parser.DInt(rng.Int63())) case ColumnType_FLOAT: return parser.NewDFloat(parser.DFloat(rng.NormFloat64())) case ColumnType_DECIMAL: d := &parser.DDecimal{} d.Dec.SetScale(inf.Scale(rng.Intn(40) - 20)) d.Dec.SetUnscaled(rng.Int63()) return d case ColumnType_DATE: return parser.NewDDate(parser.DDate(rng.Intn(10000))) case ColumnType_TIMESTAMP: return &parser.DTimestamp{Time: time.Unix(rng.Int63n(1000000), rng.Int63n(1000000))} case ColumnType_INTERVAL: return &parser.DInterval{Duration: duration.Duration{Months: rng.Int63n(1000), Days: rng.Int63n(1000), Nanos: rng.Int63n(1000000), }} case ColumnType_STRING: // Generate a random ASCII string. p := make([]byte, rng.Intn(10)) for i := range p { p[i] = byte(1 + rng.Intn(127)) } return parser.NewDString(string(p)) case ColumnType_BYTES: p := make([]byte, rng.Intn(10)) _, _ = rng.Read(p) return parser.NewDBytes(parser.DBytes(p)) case ColumnType_TIMESTAMPTZ: return &parser.DTimestampTZ{Time: time.Unix(rng.Int63n(1000000), rng.Int63n(1000000))} default: panic(fmt.Sprintf("invalid type %s", typ)) } }
// decodeOidDatum decodes bytes with specified Oid and format code into // a datum. func decodeOidDatum(id oid.Oid, code formatCode, b []byte) (parser.Datum, error) { var d parser.Datum switch id { case oid.T_bool: switch code { case formatText: v, err := strconv.ParseBool(string(b)) if err != nil { return d, err } d = parser.MakeDBool(parser.DBool(v)) default: return d, fmt.Errorf("unsupported bool format code: %d", code) } case oid.T_int2: switch code { case formatText: i, err := strconv.ParseInt(string(b), 10, 64) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) case formatBinary: var i int16 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) default: return d, fmt.Errorf("unsupported int2 format code: %d", code) } case oid.T_int4: switch code { case formatText: i, err := strconv.ParseInt(string(b), 10, 64) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) case formatBinary: var i int32 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) default: return d, fmt.Errorf("unsupported int4 format code: %d", code) } case oid.T_int8: switch code { case formatText: i, err := strconv.ParseInt(string(b), 10, 64) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) case formatBinary: var i int64 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) default: return d, fmt.Errorf("unsupported int8 format code: %d", code) } case oid.T_float4: switch code { case formatText: f, err := strconv.ParseFloat(string(b), 64) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) case formatBinary: var f float32 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) default: return d, fmt.Errorf("unsupported float4 format code: %d", code) } case oid.T_float8: switch code { case formatText: f, err := strconv.ParseFloat(string(b), 64) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) case formatBinary: var f float64 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) default: return d, fmt.Errorf("unsupported float8 format code: %d", code) } case oid.T_numeric: switch code { case formatText: dd := &parser.DDecimal{} if _, ok := dd.SetString(string(b)); !ok { return nil, fmt.Errorf("could not parse string %q as decimal", b) } d = dd default: return d, fmt.Errorf("unsupported numeric format code: %d", code) } case oid.T_text, oid.T_varchar: switch code { case formatText: d = parser.NewDString(string(b)) default: return d, fmt.Errorf("unsupported text format code: %d", code) } case oid.T_bytea: switch code { case formatText: // http://www.postgresql.org/docs/current/static/datatype-binary.html#AEN5667 // Code cribbed from github.com/lib/pq. // We only support hex encoding. if len(b) >= 2 && bytes.Equal(b[:2], []byte("\\x")) { b = b[2:] // trim off leading "\\x" result := make([]byte, hex.DecodedLen(len(b))) _, err := hex.Decode(result, b) if err != nil { return d, err } d = parser.NewDBytes(parser.DBytes(result)) } else { return d, fmt.Errorf("unsupported bytea encoding: %q", b) } case formatBinary: d = parser.NewDBytes(parser.DBytes(b)) default: return d, fmt.Errorf("unsupported bytea format code: %d", code) } case oid.T_timestamp, oid.T_timestamptz: switch code { case formatText: ts, err := parseTs(string(b)) if err != nil { return d, fmt.Errorf("could not parse string %q as timestamp", b) } d = &parser.DTimestamp{Time: ts} case formatBinary: return d, fmt.Errorf("unsupported timestamp format code: %d", code) } case oid.T_date: switch code { case formatText: ts, err := parseTs(string(b)) if err != nil { return d, fmt.Errorf("could not parse string %q as date", b) } daysSinceEpoch := ts.Unix() / secondsInDay d = parser.NewDDate(parser.DDate(daysSinceEpoch)) case formatBinary: return d, fmt.Errorf("unsupported date format code: %d", code) } default: return d, fmt.Errorf("unsupported OID: %v", id) } return d, nil }
// decodeOidDatum decodes bytes with specified Oid and format code into // a datum. func decodeOidDatum(id oid.Oid, code formatCode, b []byte) (parser.Datum, error) { var d parser.Datum switch id { case oid.T_bool: switch code { case formatText: v, err := strconv.ParseBool(string(b)) if err != nil { return d, err } d = parser.MakeDBool(parser.DBool(v)) case formatBinary: switch b[0] { case 0: d = parser.MakeDBool(false) case 1: d = parser.MakeDBool(true) default: return d, util.Errorf("unsupported binary bool: %q", b) } default: return d, util.Errorf("unsupported bool format code: %s", code) } case oid.T_int2: switch code { case formatText: i, err := strconv.ParseInt(string(b), 10, 64) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) case formatBinary: var i int16 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) default: return d, util.Errorf("unsupported int2 format code: %s", code) } case oid.T_int4: switch code { case formatText: i, err := strconv.ParseInt(string(b), 10, 64) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) case formatBinary: var i int32 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) default: return d, util.Errorf("unsupported int4 format code: %s", code) } case oid.T_int8: switch code { case formatText: i, err := strconv.ParseInt(string(b), 10, 64) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) case formatBinary: var i int64 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &i) if err != nil { return d, err } d = parser.NewDInt(parser.DInt(i)) default: return d, util.Errorf("unsupported int8 format code: %s", code) } case oid.T_float4: switch code { case formatText: f, err := strconv.ParseFloat(string(b), 64) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) case formatBinary: var f float32 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) default: return d, util.Errorf("unsupported float4 format code: %s", code) } case oid.T_float8: switch code { case formatText: f, err := strconv.ParseFloat(string(b), 64) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) case formatBinary: var f float64 err := binary.Read(bytes.NewReader(b), binary.BigEndian, &f) if err != nil { return d, err } d = parser.NewDFloat(parser.DFloat(f)) default: return d, util.Errorf("unsupported float8 format code: %s", code) } case oid.T_numeric: switch code { case formatText: dd := &parser.DDecimal{} if _, ok := dd.SetString(string(b)); !ok { return nil, util.Errorf("could not parse string %q as decimal", b) } d = dd case formatBinary: r := bytes.NewReader(b) alloc := struct { pgNum pgNumeric i16 int16 dd parser.DDecimal }{} for _, ptr := range []interface{}{ &alloc.pgNum.ndigits, &alloc.pgNum.weight, &alloc.pgNum.sign, &alloc.pgNum.dscale, } { if err := binary.Read(r, binary.BigEndian, ptr); err != nil { return d, err } } decDigits := make([]byte, 0, alloc.pgNum.ndigits*pgDecDigits) for i := int16(0); i < alloc.pgNum.ndigits-1; i++ { if err := binary.Read(r, binary.BigEndian, &alloc.i16); err != nil { return d, err } decDigits = strconv.AppendUint(decDigits, uint64(alloc.i16), 10) } // The last digit may contain padding, which we need to deal with. if err := binary.Read(r, binary.BigEndian, &alloc.i16); err != nil { return d, err } dscale := (alloc.pgNum.ndigits - 1 - alloc.pgNum.weight) * pgDecDigits if overScale := dscale - alloc.pgNum.dscale; overScale > 0 { dscale -= overScale for i := int16(0); i < overScale; i++ { alloc.i16 /= 10 } } decDigits = strconv.AppendUint(decDigits, uint64(alloc.i16), 10) alloc.dd.UnscaledBig().SetString(string(decDigits), 10) alloc.dd.SetScale(inf.Scale(dscale)) switch alloc.pgNum.sign { case pgNumericPos: case pgNumericNeg: alloc.dd.Neg(&alloc.dd.Dec) default: return d, util.Errorf("unsupported numeric sign: %s", alloc.pgNum.sign) } d = &alloc.dd default: return d, util.Errorf("unsupported numeric format code: %s", code) } case oid.T_text, oid.T_varchar: switch code { case formatText, formatBinary: d = parser.NewDString(string(b)) default: return d, util.Errorf("unsupported text format code: %s", code) } case oid.T_bytea: switch code { case formatText: // http://www.postgresql.org/docs/current/static/datatype-binary.html#AEN5667 // Code cribbed from github.com/lib/pq. // We only support hex encoding. if len(b) >= 2 && bytes.Equal(b[:2], []byte("\\x")) { b = b[2:] // trim off leading "\\x" result := make([]byte, hex.DecodedLen(len(b))) _, err := hex.Decode(result, b) if err != nil { return d, err } d = parser.NewDBytes(parser.DBytes(result)) } else { return d, util.Errorf("unsupported bytea encoding: %q", b) } case formatBinary: d = parser.NewDBytes(parser.DBytes(b)) default: return d, util.Errorf("unsupported bytea format code: %s", code) } case oid.T_timestamp, oid.T_timestamptz: switch code { case formatText: ts, err := parseTs(string(b)) if err != nil { return d, util.Errorf("could not parse string %q as timestamp", b) } d = parser.MakeDTimestamp(ts, time.Microsecond) default: return d, util.Errorf("unsupported timestamp format code: %s", code) } case oid.T_date: switch code { case formatText: ts, err := parseTs(string(b)) if err != nil { return d, util.Errorf("could not parse string %q as date", b) } daysSinceEpoch := ts.Unix() / secondsInDay d = parser.NewDDate(parser.DDate(daysSinceEpoch)) default: return d, util.Errorf("unsupported date format code: %s", code) } default: return d, util.Errorf("unsupported OID: %v", id) } return d, nil }
func dumpTable(w io.Writer, conn *sqlConn, origDBName, origTableName string) error { const limit = 100 // Escape names since they can't be used in placeholders. dbname := parser.Name(origDBName).String() tablename := parser.Name(origTableName).String() if err := conn.Exec(fmt.Sprintf("SET DATABASE = %s", dbname), nil); err != nil { return err } // Fetch all table metadata in a transaction and its time to guarantee it // doesn't change between the various SHOW statements. if err := conn.Exec("BEGIN", nil); err != nil { return err } vals, err := conn.QueryRow("SELECT cluster_logical_timestamp()::int", nil) if err != nil { return err } clusterTSStart := vals[0].(int64) clusterTS := time.Unix(0, clusterTSStart).Format(time.RFC3339Nano) // Fetch table descriptor. vals, err = conn.QueryRow(` SELECT descriptor FROM system.descriptor JOIN system.namespace tables ON tables.id = descriptor.id JOIN system.namespace dbs ON dbs.id = tables.parentid WHERE tables.name = $1 AND dbs.name = $2`, []driver.Value{origTableName, origDBName}) if err == io.EOF { return errors.Errorf("unknown database or table %s.%s", origTableName, origDBName) } else if err != nil { return err } b := vals[0].([]byte) var desc sqlbase.Descriptor if err := proto.Unmarshal(b, &desc); err != nil { return err } table := desc.GetTable() if table == nil { return errors.New("internal error: expected table descriptor") } coltypes := make(map[string]string) for _, c := range table.Columns { coltypes[c.Name] = c.Type.SQLString() } primaryIndex := table.PrimaryIndex.Name index := table.PrimaryIndex.ColumnNames indexes := strings.Join(index, ", ") // Build the SELECT query. var sbuf bytes.Buffer fmt.Fprintf(&sbuf, "SELECT %s, * FROM %s@%s AS OF SYSTEM TIME '%s'", indexes, tablename, primaryIndex, clusterTS) var wbuf bytes.Buffer fmt.Fprintf(&wbuf, " WHERE ROW (%s) > ROW (", indexes) for i := range index { if i > 0 { wbuf.WriteString(", ") } fmt.Fprintf(&wbuf, "$%d", i+1) } wbuf.WriteString(")") // No WHERE clause first time, so add a place to inject it. fmt.Fprintf(&sbuf, "%%s ORDER BY %s LIMIT %d", indexes, limit) bs := sbuf.String() vals, err = conn.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", tablename), nil) if err != nil { return err } create := vals[1].([]byte) if _, err := w.Write(create); err != nil { return err } if _, err := w.Write([]byte(";\n")); err != nil { return err } if err := conn.Exec("COMMIT", nil); err != nil { return err } // pk holds the last values of the fetched primary keys var pk []driver.Value q := fmt.Sprintf(bs, "") for { rows, err := conn.Query(q, pk) if err != nil { return err } cols := rows.Columns() pkcols := cols[:len(index)] cols = cols[len(index):] inserts := make([][]string, 0, limit) i := 0 for i < limit { vals := make([]driver.Value, len(cols)+len(pkcols)) if err := rows.Next(vals); err == io.EOF { break } else if err != nil { return err } if pk == nil { q = fmt.Sprintf(bs, wbuf.String()) } pk = vals[:len(index)] vals = vals[len(index):] ivals := make([]string, len(vals)) // Values need to be correctly encoded for INSERT statements in a text file. for si, sv := range vals { switch t := sv.(type) { case nil: ivals[si] = "NULL" case bool: ivals[si] = parser.MakeDBool(parser.DBool(t)).String() case int64: ivals[si] = parser.NewDInt(parser.DInt(t)).String() case float64: ivals[si] = parser.NewDFloat(parser.DFloat(t)).String() case []byte: switch ct := coltypes[cols[si]]; ct { case "INTERVAL": ivals[si] = fmt.Sprintf("'%s'", t) case "DECIMAL": ivals[si] = fmt.Sprintf("%s", t) default: // STRING and BYTES types can have optional length suffixes, so only examine // the prefix of the type. if strings.HasPrefix(coltypes[cols[si]], "STRING") { ivals[si] = parser.NewDString(string(t)).String() } else if strings.HasPrefix(coltypes[cols[si]], "BYTES") { ivals[si] = parser.NewDBytes(parser.DBytes(t)).String() } else { panic(errors.Errorf("unknown []byte type: %s, %v: %s", t, cols[si], coltypes[cols[si]])) } } case time.Time: var d parser.Datum ct := coltypes[cols[si]] switch ct { case "DATE": d = parser.NewDDateFromTime(t, time.UTC) case "TIMESTAMP": d = parser.MakeDTimestamp(t, time.Nanosecond) case "TIMESTAMP WITH TIME ZONE": d = parser.MakeDTimestampTZ(t, time.Nanosecond) default: panic(errors.Errorf("unknown timestamp type: %s, %v: %s", t, cols[si], coltypes[cols[si]])) } ivals[si] = fmt.Sprintf("'%s'", d) default: panic(errors.Errorf("unknown field type: %T (%s)", t, cols[si])) } } inserts = append(inserts, ivals) i++ } for si, sv := range pk { b, ok := sv.([]byte) if ok && strings.HasPrefix(coltypes[pkcols[si]], "STRING") { // Primary key strings need to be converted to a go string, but not SQL // encoded since they aren't being written to a text file. pk[si] = string(b) } } if err := rows.Close(); err != nil { return err } if i == 0 { break } fmt.Fprintf(w, "\nINSERT INTO %s VALUES", tablename) for idx, values := range inserts { if idx > 0 { fmt.Fprint(w, ",") } fmt.Fprint(w, "\n\t(") for vi, v := range values { if vi > 0 { fmt.Fprint(w, ", ") } fmt.Fprint(w, v) } fmt.Fprint(w, ")") } fmt.Fprintln(w, ";") if i < limit { break } } return nil }
func TestIndexKey(t *testing.T) { rng, _ := randutil.NewPseudoRand() var a DatumAlloc tests := []indexKeyTest{ {nil, nil, []parser.Datum{parser.NewDInt(10)}, []parser.Datum{parser.NewDInt(20)}, }, {[]ID{100}, nil, []parser.Datum{parser.NewDInt(10), parser.NewDInt(11)}, []parser.Datum{parser.NewDInt(20)}, }, {[]ID{100, 200}, nil, []parser.Datum{parser.NewDInt(10), parser.NewDInt(11), parser.NewDInt(12)}, []parser.Datum{parser.NewDInt(20)}, }, {nil, []ID{100}, []parser.Datum{parser.NewDInt(10)}, []parser.Datum{parser.NewDInt(20), parser.NewDInt(21)}, }, {[]ID{100}, []ID{100}, []parser.Datum{parser.NewDInt(10), parser.NewDInt(11)}, []parser.Datum{parser.NewDInt(20), parser.NewDInt(21)}, }, {[]ID{100}, []ID{200}, []parser.Datum{parser.NewDInt(10), parser.NewDInt(11)}, []parser.Datum{parser.NewDInt(20), parser.NewDInt(21)}, }, {[]ID{100, 200}, []ID{100, 300}, []parser.Datum{parser.NewDInt(10), parser.NewDInt(11), parser.NewDInt(12)}, []parser.Datum{parser.NewDInt(20), parser.NewDInt(21), parser.NewDInt(22)}, }, } for i := 0; i < 1000; i++ { var t indexKeyTest t.primaryInterleaves = make([]ID, rng.Intn(10)) for j := range t.primaryInterleaves { t.primaryInterleaves[j] = ID(1 + rng.Intn(10)) } valuesLen := randutil.RandIntInRange(rng, len(t.primaryInterleaves)+1, len(t.primaryInterleaves)+10) t.primaryValues = make([]parser.Datum, valuesLen) for j := range t.primaryValues { t.primaryValues[j] = RandDatum(rng, ColumnType_INT, true) } t.secondaryInterleaves = make([]ID, rng.Intn(10)) for j := range t.secondaryInterleaves { t.secondaryInterleaves[j] = ID(1 + rng.Intn(10)) } valuesLen = randutil.RandIntInRange(rng, len(t.secondaryInterleaves)+1, len(t.secondaryInterleaves)+10) t.secondaryValues = make([]parser.Datum, valuesLen) for j := range t.secondaryValues { t.secondaryValues[j] = RandDatum(rng, ColumnType_INT, true) } tests = append(tests, t) } for i, test := range tests { tableDesc, colMap := makeTableDescForTest(test) testValues := append(test.primaryValues, test.secondaryValues...) primaryKeyPrefix := MakeIndexKeyPrefix(&tableDesc, tableDesc.PrimaryIndex.ID) primaryKey, _, err := EncodeIndexKey( &tableDesc, &tableDesc.PrimaryIndex, colMap, testValues, primaryKeyPrefix) if err != nil { t.Fatal(err) } primaryValue := roachpb.MakeValueFromBytes(nil) primaryIndexKV := client.KeyValue{Key: primaryKey, Value: &primaryValue} secondaryIndexEntry, err := EncodeSecondaryIndex( &tableDesc, &tableDesc.Indexes[0], colMap, testValues) if err != nil { t.Fatal(err) } secondaryIndexKV := client.KeyValue{ Key: secondaryIndexEntry.Key, Value: &secondaryIndexEntry.Value, } checkEntry := func(index *IndexDescriptor, entry client.KeyValue) { values, err := decodeIndex(&a, &tableDesc, index, entry.Key) if err != nil { t.Fatal(err) } for j, value := range values { testValue := testValues[colMap[index.ColumnIDs[j]]] if value.Compare(testValue) != 0 { t.Fatalf("%d: value %d got %q but expected %q", i, j, value, testValue) } } indexID, _, err := DecodeIndexKeyPrefix(&a, &tableDesc, entry.Key) if err != nil { t.Fatal(err) } if indexID != index.ID { t.Errorf("%d", i) } extracted, err := ExtractIndexKey(&a, &tableDesc, entry) if err != nil { t.Fatal(err) } if !bytes.Equal(extracted, primaryKey) { t.Errorf("%d got %s <%x>, but expected %s <%x>", i, extracted, []byte(extracted), roachpb.Key(primaryKey), primaryKey) } } checkEntry(&tableDesc.PrimaryIndex, primaryIndexKV) checkEntry(&tableDesc.Indexes[0], secondaryIndexKV) } }
func TestTableReader(t *testing.T) { defer leaktest.AfterTest(t)() _, sqlDB, kvDB, cleanup := sqlutils.SetupServer(t) defer cleanup() // Create a table where each row is: // // | a | b | sum | s | // |-----------------------------------------------------------------| // | rowId/10 | rowId%10 | rowId/10 + rowId%10 | IntToEnglish(rowId) | aFn := func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row / 10)) } bFn := func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row % 10)) } sumFn := func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row/10 + row%10)) } sqlutils.CreateTable(t, sqlDB, "t", "a INT, b INT, sum INT, s STRING, PRIMARY KEY (a,b), INDEX bs (b,s)", 99, sqlutils.ToRowFn(aFn, bFn, sumFn, sqlutils.RowEnglishFn)) td := sqlbase.GetTableDescriptor(kvDB, "test", "t") makeIndexSpan := func(start, end int) TableReaderSpan { var span roachpb.Span prefix := roachpb.Key(sqlbase.MakeIndexKeyPrefix(td.ID, td.Indexes[0].ID)) span.Key = append(prefix, encoding.EncodeVarintAscending(nil, int64(start))...) span.EndKey = append(span.EndKey, prefix...) span.EndKey = append(span.EndKey, encoding.EncodeVarintAscending(nil, int64(end))...) return TableReaderSpan{Span: span} } testCases := []struct { spec TableReaderSpec expected string }{ { spec: TableReaderSpec{ Filter: Expression{Expr: "$2 < 5 AND $1 != 3"}, // sum < 5 && b != 3 OutputColumns: []uint32{0, 1}, }, expected: "[[0 1] [0 2] [0 4] [1 0] [1 1] [1 2] [2 0] [2 1] [2 2] [3 0] [3 1] [4 0]]", }, { spec: TableReaderSpec{ Filter: Expression{Expr: "$2 < 5 AND $1 != 3"}, OutputColumns: []uint32{3}, // s HardLimit: 4, }, expected: "[['one'] ['two'] ['four'] ['one-zero']]", }, { spec: TableReaderSpec{ IndexIdx: 1, Reverse: true, Spans: []TableReaderSpan{makeIndexSpan(4, 6)}, Filter: Expression{Expr: "$0 < 3"}, // sum < 8 OutputColumns: []uint32{0, 1}, SoftLimit: 1, }, expected: "[[2 5] [1 5] [0 5] [2 4] [1 4] [0 4]]", }, } for _, c := range testCases { ts := c.spec ts.Table = *td txn := client.NewTxn(context.Background(), *kvDB) out := &RowBuffer{} tr, err := newTableReader(&ts, txn, out, &parser.EvalContext{}) if err != nil { t.Fatal(err) } tr.Run(nil) if out.err != nil { t.Fatal(out.err) } if !out.closed { t.Fatalf("output RowReceiver not closed") } if result := out.rows.String(); result != c.expected { t.Errorf("invalid results: %s, expected %s'", result, c.expected) } } }
func dIntFnOrNull(fn func() (int32, bool)) parser.Datum { if n, ok := fn(); ok { return parser.NewDInt(parser.DInt(n)) } return parser.DNull }
func TestEncDatum(t *testing.T) { a := &DatumAlloc{} x := &EncDatum{} if !x.IsUnset() { t.Errorf("empty EncDatum should be unset") } if _, ok := x.Encoding(); ok { t.Errorf("empty EncDatum has an encoding") } x.SetDatum(ColumnType_INT, parser.NewDInt(5)) if x.IsUnset() { t.Errorf("unset after SetDatum()") } encoded, err := x.Encode(a, DatumEncoding_ASCENDING_KEY, nil) if err != nil { t.Fatal(err) } y := &EncDatum{} y.SetEncoded(ColumnType_INT, DatumEncoding_ASCENDING_KEY, encoded) if y.IsUnset() { t.Errorf("unset after SetEncoded()") } if enc, ok := y.Encoding(); !ok { t.Error("no encoding after SetEncoded") } else if enc != DatumEncoding_ASCENDING_KEY { t.Errorf("invalid encoding %d", enc) } err = y.Decode(a) if err != nil { t.Fatal(err) } if cmp := y.Datum.Compare(x.Datum); cmp != 0 { t.Errorf("Datums should be equal, cmp = %d", cmp) } enc2, err := y.Encode(a, DatumEncoding_DESCENDING_KEY, nil) if err != nil { t.Fatal(err) } // y's encoding should not change. if enc, ok := y.Encoding(); !ok { t.Error("no encoding") } else if enc != DatumEncoding_ASCENDING_KEY { t.Errorf("invalid encoding %d", enc) } x.SetEncoded(ColumnType_INT, DatumEncoding_DESCENDING_KEY, enc2) if enc, ok := x.Encoding(); !ok { t.Error("no encoding") } else if enc != DatumEncoding_DESCENDING_KEY { t.Errorf("invalid encoding %d", enc) } err = x.Decode(a) if err != nil { t.Fatal(err) } if cmp := y.Datum.Compare(x.Datum); cmp != 0 { t.Errorf("Datums should be equal, cmp = %d", cmp) } }
func TestUnorderedSync(t *testing.T) { mrc := &MultiplexedRowChannel{} mrc.Init(5) for i := 1; i <= 5; i++ { go func(i int) { for j := 1; j <= 100; j++ { var a, b sqlbase.EncDatum a.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(i))) b.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(j))) row := sqlbase.EncDatumRow{a, b} mrc.PushRow(row) } mrc.Close(nil) }(i) } var retRows sqlbase.EncDatumRows for { row, err := mrc.NextRow() if err != nil { t.Fatal(err) } if row == nil { break } retRows = append(retRows, row) } // Verify all elements. for i := 1; i <= 5; i++ { j := 1 for _, row := range retRows { if int(*(row[0].Datum.(*parser.DInt))) == i { if int(*(row[1].Datum.(*parser.DInt))) != j { t.Errorf("Expected [%d %d], got %s", i, j, row) } j++ } } if j != 101 { t.Errorf("Missing [%d %d]", i, j) } } // Test case when one source closes with an error. mrc = &MultiplexedRowChannel{} mrc.Init(5) for i := 1; i <= 5; i++ { go func(i int) { for j := 1; j <= 100; j++ { var a, b sqlbase.EncDatum a.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(i))) b.SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(j))) row := sqlbase.EncDatumRow{a, b} mrc.PushRow(row) } var err error if i == 3 { err = fmt.Errorf("Test error") } mrc.Close(err) }(i) } for { row, err := mrc.NextRow() if err != nil { if err.Error() != "Test error" { t.Error(err) } break } if row == nil { t.Error("Did not receive expected error") } } }
func TestJoinReader(t *testing.T) { defer leaktest.AfterTest(t)() s, sqlDB, kvDB := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() // Create a table where each row is: // // | a | b | sum | s | // |-----------------------------------------------------------------| // | rowId/10 | rowId%10 | rowId/10 + rowId%10 | IntToEnglish(rowId) | aFn := func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row / 10)) } bFn := func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row % 10)) } sumFn := func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row/10 + row%10)) } sqlutils.CreateTable(t, sqlDB, "t", "a INT, b INT, sum INT, s STRING, PRIMARY KEY (a,b), INDEX bs (b,s)", 99, sqlutils.ToRowFn(aFn, bFn, sumFn, sqlutils.RowEnglishFn)) td := sqlbase.GetTableDescriptor(kvDB, "test", "t") testCases := []struct { spec JoinReaderSpec input [][]parser.Datum expected string }{ { spec: JoinReaderSpec{ OutputColumns: []uint32{0, 1, 2}, }, input: [][]parser.Datum{ {aFn(2), bFn(2)}, {aFn(5), bFn(5)}, {aFn(10), bFn(10)}, {aFn(15), bFn(15)}, }, expected: "[[0 2 2] [0 5 5] [1 0 1] [1 5 6]]", }, { spec: JoinReaderSpec{ Filter: Expression{Expr: "$2 <= 5"}, // sum <= 5 OutputColumns: []uint32{3}, }, input: [][]parser.Datum{ {aFn(1), bFn(1)}, {aFn(25), bFn(25)}, {aFn(5), bFn(5)}, {aFn(21), bFn(21)}, {aFn(34), bFn(34)}, {aFn(13), bFn(13)}, {aFn(51), bFn(51)}, {aFn(50), bFn(50)}, }, expected: "[['one'] ['five'] ['two-one'] ['one-three'] ['five-zero']]", }, } for _, c := range testCases { js := c.spec js.Table = *td txn := client.NewTxn(context.Background(), *kvDB) in := &RowBuffer{} for _, row := range c.input { encRow := make(sqlbase.EncDatumRow, len(row)) for i, d := range row { encRow[i].SetDatum(sqlbase.ColumnType_INT, d) } in.rows = append(in.rows, encRow) } out := &RowBuffer{} jr, err := newJoinReader(&js, txn, in, out, &parser.EvalContext{}) if err != nil { t.Fatal(err) } jr.Run(nil) if out.err != nil { t.Fatal(out.err) } if !in.done { t.Fatal("joinReader stopped accepting rows") } if !out.closed { t.Fatalf("output RowReceiver not closed") } if result := out.rows.String(); result != c.expected { t.Errorf("invalid results: %s, expected %s'", result, c.expected) } } }
func TestAdminAPITableDetailsZone(t *testing.T) { defer leaktest.AfterTest(t)() s, _, _ := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) // Create database and table. session := sql.NewSession(sql.SessionArgs{User: security.RootUser}, ts.sqlExecutor, nil) setupQueries := []string{ "CREATE DATABASE test", "CREATE TABLE test.tbl (val STRING)", } for _, q := range setupQueries { res := ts.sqlExecutor.ExecuteStatements(context.Background(), session, q, nil) if res.ResultList[0].Err != nil { t.Fatalf("error executing '%s': %s", q, res.ResultList[0].Err) } } // Function to verify the zone for test.tbl as returned by the Admin API. verifyZone := func(expectedZone config.ZoneConfig, expectedLevel serverpb.ZoneConfigurationLevel) { var resp serverpb.TableDetailsResponse if err := apiGet(s, "databases/test/tables/tbl", &resp); err != nil { t.Fatal(err) } if a, e := &resp.ZoneConfig, &expectedZone; !proto.Equal(a, e) { t.Errorf("actual zone config %v did not match expected value %v", a, e) } if a, e := resp.ZoneConfigLevel, expectedLevel; a != e { t.Errorf("actual ZoneConfigurationLevel %s did not match expected value %s", a, e) } if t.Failed() { t.FailNow() } } // Function to store a zone config for a given object ID. setZone := func(zoneCfg config.ZoneConfig, id sqlbase.ID) { zoneBytes, err := zoneCfg.Marshal() if err != nil { t.Fatal(err) } const query = `INSERT INTO system.zones VALUES($1, $2)` params := parser.NewPlaceholderInfo() params.SetValue(`1`, parser.NewDInt(parser.DInt(id))) params.SetValue(`2`, parser.NewDBytes(parser.DBytes(zoneBytes))) res := ts.sqlExecutor.ExecuteStatements(context.Background(), session, query, params) if res.ResultList[0].Err != nil { t.Fatalf("error executing '%s': %s", query, res.ResultList[0].Err) } } // Verify zone matches cluster default. verifyZone(config.DefaultZoneConfig(), serverpb.ZoneConfigurationLevel_CLUSTER) // Get ID path for table. This will be an array of three IDs, containing the ID of the root namespace, // the database, and the table (in that order). idPath, err := ts.admin.queryDescriptorIDPath(context.Background(), session, []string{"test", "tbl"}) if err != nil { t.Fatal(err) } // Apply zone configuration to database and check again. dbZone := config.ZoneConfig{ RangeMinBytes: 456, } setZone(dbZone, idPath[1]) verifyZone(dbZone, serverpb.ZoneConfigurationLevel_DATABASE) // Apply zone configuration to table and check again. tblZone := config.ZoneConfig{ RangeMinBytes: 789, } setZone(tblZone, idPath[2]) verifyZone(tblZone, serverpb.ZoneConfigurationLevel_TABLE) }
func TestValues(t *testing.T) { defer leaktest.AfterTest(t)() p := makePlanner() vInt := int64(5) vNum := 3.14159 vStr := "two furs one cub" vBool := true unsupp := &parser.RangeCond{} intVal := func(v int64) *parser.NumVal { return &parser.NumVal{Value: constant.MakeInt64(v)} } floatVal := func(f float64) *parser.CastExpr { return &parser.CastExpr{ Expr: &parser.NumVal{Value: constant.MakeFloat64(f)}, Type: &parser.FloatColType{}, } } asRow := func(datums ...parser.Datum) []parser.DTuple { return []parser.DTuple{datums} } makeValues := func(tuples ...*parser.Tuple) *parser.ValuesClause { return &parser.ValuesClause{Tuples: tuples} } makeTuple := func(exprs ...parser.Expr) *parser.Tuple { return &parser.Tuple{Exprs: exprs} } testCases := []struct { stmt *parser.ValuesClause rows []parser.DTuple ok bool }{ { makeValues(makeTuple(intVal(vInt))), asRow(parser.NewDInt(parser.DInt(vInt))), true, }, { makeValues(makeTuple(intVal(vInt), intVal(vInt))), asRow(parser.NewDInt(parser.DInt(vInt)), parser.NewDInt(parser.DInt(vInt))), true, }, { makeValues(makeTuple(floatVal(vNum))), asRow(parser.NewDFloat(parser.DFloat(vNum))), true, }, { makeValues(makeTuple(parser.NewDString(vStr))), asRow(parser.NewDString(vStr)), true, }, { makeValues(makeTuple(parser.NewDBytes(parser.DBytes(vStr)))), asRow(parser.NewDBytes(parser.DBytes(vStr))), true, }, { makeValues(makeTuple(parser.MakeDBool(parser.DBool(vBool)))), asRow(parser.MakeDBool(parser.DBool(vBool))), true, }, { makeValues(makeTuple(unsupp)), nil, false, }, } for i, tc := range testCases { plan, err := func() (_ planNode, err error) { defer func() { if r := recover(); r != nil { err = errors.Errorf("%v", r) } }() return p.ValuesClause(tc.stmt, nil) }() if err == nil != tc.ok { t.Errorf("%d: error_expected=%t, but got error %v", i, tc.ok, err) } if plan != nil { if err := plan.expandPlan(); err != nil { t.Errorf("%d: unexpected error in expandPlan: %v", i, err) continue } if err := plan.Start(); err != nil { t.Errorf("%d: unexpected error in Start: %v", i, err) continue } var rows []parser.DTuple next, err := plan.Next() for ; next; next, err = plan.Next() { rows = append(rows, plan.Values()) } if err != nil { t.Error(err) continue } if !reflect.DeepEqual(rows, tc.rows) { t.Errorf("%d: expected rows:\n%+v\nactual rows:\n%+v", i, tc.rows, rows) } } } }
func dumpTable(w io.Writer, conn *sqlConn, origDBName, origTableName string) error { const limit = 100 // Escape names since they can't be used in placeholders. dbname := parser.Name(origDBName).String() tablename := parser.Name(origTableName).String() if err := conn.Exec(fmt.Sprintf("SET DATABASE = %s", dbname), nil); err != nil { return err } // Fetch all table metadata in a transaction and its time to guarantee it // doesn't change between the various SHOW statements. if err := conn.Exec("BEGIN", nil); err != nil { return err } vals, err := conn.QueryRow("SELECT cluster_logical_timestamp()::int", nil) if err != nil { return err } clusterTSStart := vals[0].(int64) clusterTS := time.Unix(0, clusterTSStart).Format(time.RFC3339Nano) // A previous version of the code did a SELECT on system.descriptor. This // required the SELECT privilege to the descriptor table, which only root // has. Allowing non-root to do this would let users see other users' table // descriptors which is a problem in multi-tenancy. // Fetch column types. rows, err := conn.Query(fmt.Sprintf("SHOW COLUMNS FROM %s", tablename), nil) if err != nil { return err } vals = make([]driver.Value, 2) coltypes := make(map[string]string) for { if err := rows.Next(vals); err == io.EOF { break } else if err != nil { return err } name, ok := vals[0].(string) if !ok { return fmt.Errorf("unexpected value: %T", vals[1]) } typ, ok := vals[1].(string) if !ok { return fmt.Errorf("unexpected value: %T", vals[4]) } coltypes[name] = typ } if err := rows.Close(); err != nil { return err } // index holds the names, in order, of the primary key columns. var index []string // Primary index is always the first index returned by SHOW INDEX. rows, err = conn.Query(fmt.Sprintf("SHOW INDEX FROM %s", tablename), nil) if err != nil { return err } vals = make([]driver.Value, 5) var primaryIndex string // Find the primary index columns. for { if err := rows.Next(vals); err == io.EOF { break } else if err != nil { return err } b, ok := vals[1].(string) if !ok { return fmt.Errorf("unexpected value: %T", vals[1]) } if primaryIndex == "" { primaryIndex = b } else if primaryIndex != b { break } b, ok = vals[4].(string) if !ok { return fmt.Errorf("unexpected value: %T", vals[4]) } index = append(index, parser.Name(b).String()) } if err := rows.Close(); err != nil { return err } if len(index) == 0 { return fmt.Errorf("no primary key index found") } indexes := strings.Join(index, ", ") // Build the SELECT query. var sbuf bytes.Buffer fmt.Fprintf(&sbuf, "SELECT %s, * FROM %s@%s AS OF SYSTEM TIME '%s'", indexes, tablename, primaryIndex, clusterTS) var wbuf bytes.Buffer fmt.Fprintf(&wbuf, " WHERE ROW (%s) > ROW (", indexes) for i := range index { if i > 0 { wbuf.WriteString(", ") } fmt.Fprintf(&wbuf, "$%d", i+1) } wbuf.WriteString(")") // No WHERE clause first time, so add a place to inject it. fmt.Fprintf(&sbuf, "%%s ORDER BY %s LIMIT %d", indexes, limit) bs := sbuf.String() vals, err = conn.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", tablename), nil) if err != nil { return err } create := vals[1].(string) if _, err := w.Write([]byte(create)); err != nil { return err } if _, err := w.Write([]byte(";\n")); err != nil { return err } if err := conn.Exec("COMMIT", nil); err != nil { return err } // pk holds the last values of the fetched primary keys var pk []driver.Value q := fmt.Sprintf(bs, "") for { rows, err := conn.Query(q, pk) if err != nil { return err } cols := rows.Columns() pkcols := cols[:len(index)] cols = cols[len(index):] inserts := make([][]string, 0, limit) i := 0 for i < limit { vals := make([]driver.Value, len(cols)+len(pkcols)) if err := rows.Next(vals); err == io.EOF { break } else if err != nil { return err } if pk == nil { q = fmt.Sprintf(bs, wbuf.String()) } pk = vals[:len(index)] vals = vals[len(index):] ivals := make([]string, len(vals)) // Values need to be correctly encoded for INSERT statements in a text file. for si, sv := range vals { switch t := sv.(type) { case nil: ivals[si] = "NULL" case bool: ivals[si] = parser.MakeDBool(parser.DBool(t)).String() case int64: ivals[si] = parser.NewDInt(parser.DInt(t)).String() case float64: ivals[si] = parser.NewDFloat(parser.DFloat(t)).String() case string: ivals[si] = parser.NewDString(t).String() case []byte: switch ct := coltypes[cols[si]]; ct { case "INTERVAL": ivals[si] = fmt.Sprintf("'%s'", t) case "BYTES": ivals[si] = parser.NewDBytes(parser.DBytes(t)).String() default: // STRING and DECIMAL types can have optional length // suffixes, so only examine the prefix of the type. if strings.HasPrefix(coltypes[cols[si]], "STRING") { ivals[si] = parser.NewDString(string(t)).String() } else if strings.HasPrefix(coltypes[cols[si]], "DECIMAL") { ivals[si] = string(t) } else { panic(errors.Errorf("unknown []byte type: %s, %v: %s", t, cols[si], coltypes[cols[si]])) } } case time.Time: var d parser.Datum ct := coltypes[cols[si]] switch ct { case "DATE": d = parser.NewDDateFromTime(t, time.UTC) case "TIMESTAMP": d = parser.MakeDTimestamp(t, time.Nanosecond) case "TIMESTAMP WITH TIME ZONE": d = parser.MakeDTimestampTZ(t, time.Nanosecond) default: panic(errors.Errorf("unknown timestamp type: %s, %v: %s", t, cols[si], coltypes[cols[si]])) } ivals[si] = fmt.Sprintf("'%s'", d) default: panic(errors.Errorf("unknown field type: %T (%s)", t, cols[si])) } } inserts = append(inserts, ivals) i++ } for si, sv := range pk { b, ok := sv.([]byte) if ok && strings.HasPrefix(coltypes[pkcols[si]], "STRING") { // Primary key strings need to be converted to a go string, but not SQL // encoded since they aren't being written to a text file. pk[si] = string(b) } } if err := rows.Close(); err != nil { return err } if i == 0 { break } fmt.Fprintf(w, "\nINSERT INTO %s VALUES", tablename) for idx, values := range inserts { if idx > 0 { fmt.Fprint(w, ",") } fmt.Fprint(w, "\n\t(") for vi, v := range values { if vi > 0 { fmt.Fprint(w, ", ") } fmt.Fprint(w, v) } fmt.Fprint(w, ")") } fmt.Fprintln(w, ";") if i < limit { break } } return nil }
populate: func(p *planner, addRow func(...parser.Datum)) error { return forEachTableDesc(p, func(db *sqlbase.DatabaseDescriptor, table *sqlbase.TableDescriptor) { // Table descriptors already holds columns in-order. visible := 0 for _, column := range table.Columns { if column.Hidden { continue } visible++ addRow( defString, // table_catalog parser.NewDString(db.Name), // table_schema parser.NewDString(table.Name), // table_name parser.NewDString(column.Name), // column_name parser.NewDInt(parser.DInt(visible)), // ordinal_position, 1-indexed dStringOrNull(column.DefaultExpr), // column_default yesOrNoDatum(column.Nullable), // is_nullable parser.NewDString(column.Type.Kind.String()), // data_type characterMaximumLength(column.Type), // character_maximum_length characterOctetLength(column.Type), // character_octet_length numericPrecision(column.Type), // numeric_precision numericScale(column.Type), // numeric_scale datetimePrecision(column.Type), // datetime_precision ) } }, ) }, }
func TestClusterFlow(t *testing.T) { defer leaktest.AfterTest(t)() const numRows = 100 args := base.TestClusterArgs{ReplicationMode: base.ReplicationManual} tc := serverutils.StartTestCluster(t, 3, args) defer tc.Stopper().Stop() sumDigitsFn := func(row int) parser.Datum { sum := 0 for row > 0 { sum += row % 10 row /= 10 } return parser.NewDInt(parser.DInt(sum)) } sqlutils.CreateTable(t, tc.ServerConn(0), "t", "num INT PRIMARY KEY, digitsum INT, numstr STRING, INDEX s (digitsum)", numRows, sqlutils.ToRowFn(sqlutils.RowIdxFn, sumDigitsFn, sqlutils.RowEnglishFn)) kvDB := tc.Server(0).KVClient().(*client.DB) desc := sqlbase.GetTableDescriptor(kvDB, "test", "t") makeIndexSpan := func(start, end int) TableReaderSpan { var span roachpb.Span prefix := roachpb.Key(sqlbase.MakeIndexKeyPrefix(desc, desc.Indexes[0].ID)) span.Key = append(prefix, encoding.EncodeVarintAscending(nil, int64(start))...) span.EndKey = append(span.EndKey, prefix...) span.EndKey = append(span.EndKey, encoding.EncodeVarintAscending(nil, int64(end))...) return TableReaderSpan{Span: span} } // Set up table readers on three hosts feeding data into a join reader on // the third host. This is a basic test for the distributed flow // infrastructure, including local and remote streams. // // Note that the ranges won't necessarily be local to the table readers, but // that doesn't matter for the purposes of this test. tr1 := TableReaderSpec{ Table: *desc, IndexIdx: 1, OutputColumns: []uint32{0, 1}, Spans: []TableReaderSpan{makeIndexSpan(0, 8)}, } tr2 := TableReaderSpec{ Table: *desc, IndexIdx: 1, OutputColumns: []uint32{0, 1}, Spans: []TableReaderSpan{makeIndexSpan(8, 12)}, } tr3 := TableReaderSpec{ Table: *desc, IndexIdx: 1, OutputColumns: []uint32{0, 1}, Spans: []TableReaderSpan{makeIndexSpan(12, 100)}, } jr := JoinReaderSpec{ Table: *desc, OutputColumns: []uint32{2}, } txn := client.NewTxn(context.Background(), *kvDB) fid := FlowID{uuid.MakeV4()} req1 := &SetupFlowRequest{Txn: txn.Proto} req1.Flow = FlowSpec{ FlowID: fid, Processors: []ProcessorSpec{{ Core: ProcessorCoreUnion{TableReader: &tr1}, Output: []OutputRouterSpec{{ Type: OutputRouterSpec_MIRROR, Streams: []StreamEndpointSpec{ {Mailbox: &MailboxSpec{StreamID: 0, TargetAddr: tc.Server(2).ServingAddr()}}, }, }}, }}, } req2 := &SetupFlowRequest{Txn: txn.Proto} req2.Flow = FlowSpec{ FlowID: fid, Processors: []ProcessorSpec{{ Core: ProcessorCoreUnion{TableReader: &tr2}, Output: []OutputRouterSpec{{ Type: OutputRouterSpec_MIRROR, Streams: []StreamEndpointSpec{ {Mailbox: &MailboxSpec{StreamID: 1, TargetAddr: tc.Server(2).ServingAddr()}}, }, }}, }}, } req3 := &SetupFlowRequest{Txn: txn.Proto} req3.Flow = FlowSpec{ FlowID: fid, Processors: []ProcessorSpec{ { Core: ProcessorCoreUnion{TableReader: &tr3}, Output: []OutputRouterSpec{{ Type: OutputRouterSpec_MIRROR, Streams: []StreamEndpointSpec{ {LocalStreamID: LocalStreamID(0)}, }, }}, }, { Input: []InputSyncSpec{{ Type: InputSyncSpec_ORDERED, Ordering: Ordering{Columns: []Ordering_Column{{1, Ordering_Column_ASC}}}, Streams: []StreamEndpointSpec{ {Mailbox: &MailboxSpec{StreamID: 0}}, {Mailbox: &MailboxSpec{StreamID: 1}}, {LocalStreamID: LocalStreamID(0)}, }, }}, Core: ProcessorCoreUnion{JoinReader: &jr}, Output: []OutputRouterSpec{{ Type: OutputRouterSpec_MIRROR, Streams: []StreamEndpointSpec{{Mailbox: &MailboxSpec{SimpleResponse: true}}}, }}}, }, } var clients []DistSQLClient for i := 0; i < 3; i++ { s := tc.Server(i) conn, err := s.RPCContext().GRPCDial(s.ServingAddr()) if err != nil { t.Fatal(err) } clients = append(clients, NewDistSQLClient(conn)) } ctx := context.Background() if log.V(1) { log.Infof(ctx, "Setting up flow on 0") } if resp, err := clients[0].SetupFlow(context.Background(), req1); err != nil { t.Fatal(err) } else if resp.Error != nil { t.Fatal(resp.Error) } if log.V(1) { log.Infof(ctx, "Setting up flow on 1") } if resp, err := clients[1].SetupFlow(context.Background(), req2); err != nil { t.Fatal(err) } else if resp.Error != nil { t.Fatal(resp.Error) } if log.V(1) { log.Infof(ctx, "Running flow on 2") } stream, err := clients[2].RunSimpleFlow(context.Background(), req3) if err != nil { t.Fatal(err) } var decoder StreamDecoder var rows sqlbase.EncDatumRows for { msg, err := stream.Recv() if err != nil { if err == io.EOF { break } t.Fatal(err) } err = decoder.AddMessage(msg) if err != nil { t.Fatal(err) } rows = testGetDecodedRows(t, &decoder, rows) } if done, trailerErr := decoder.IsDone(); !done { t.Fatal("stream not done") } else if trailerErr != nil { t.Fatal("error in the stream trailer:", trailerErr) } // The result should be all the numbers in string form, ordered by the // digit sum (and then by number). var results []string for sum := 1; sum <= 50; sum++ { for i := 1; i <= numRows; i++ { if int(*sumDigitsFn(i).(*parser.DInt)) == sum { results = append(results, fmt.Sprintf("['%s']", sqlutils.IntToEnglish(i))) } } } expected := strings.Join(results, " ") expected = "[" + expected + "]" if rowStr := rows.String(); rowStr != expected { t.Errorf("Result: %s\n Expected: %s\n", rowStr, expected) } }
func TestOrderedSync(t *testing.T) { v := [6]sqlbase.EncDatum{} for i := range v { v[i].SetDatum(sqlbase.ColumnType_INT, parser.NewDInt(parser.DInt(i))) } asc := encoding.Ascending desc := encoding.Descending testCases := []struct { sources []sqlbase.EncDatumRows ordering sqlbase.ColumnOrdering expected sqlbase.EncDatumRows }{ { sources: []sqlbase.EncDatumRows{ { {v[0], v[1], v[4]}, {v[0], v[1], v[2]}, {v[0], v[2], v[3]}, {v[1], v[1], v[3]}, }, { {v[1], v[0], v[4]}, }, { {v[0], v[0], v[0]}, {v[4], v[4], v[4]}, }, }, ordering: sqlbase.ColumnOrdering{ {ColIdx: 0, Direction: asc}, {ColIdx: 1, Direction: asc}, }, expected: sqlbase.EncDatumRows{ {v[0], v[0], v[0]}, {v[0], v[1], v[4]}, {v[0], v[1], v[2]}, {v[0], v[2], v[3]}, {v[1], v[0], v[4]}, {v[1], v[1], v[3]}, {v[4], v[4], v[4]}, }, }, { sources: []sqlbase.EncDatumRows{ {}, { {v[1], v[0], v[4]}, }, { {v[3], v[4], v[1]}, {v[4], v[4], v[4]}, {v[3], v[2], v[0]}, }, { {v[4], v[4], v[5]}, {v[3], v[3], v[0]}, {v[0], v[0], v[0]}, }, }, ordering: sqlbase.ColumnOrdering{ {ColIdx: 1, Direction: desc}, {ColIdx: 0, Direction: asc}, {ColIdx: 2, Direction: asc}, }, expected: sqlbase.EncDatumRows{ {v[3], v[4], v[1]}, {v[4], v[4], v[4]}, {v[4], v[4], v[5]}, {v[3], v[3], v[0]}, {v[3], v[2], v[0]}, {v[0], v[0], v[0]}, {v[1], v[0], v[4]}, }, }, } for testIdx, c := range testCases { var sources []RowSource for _, srcRows := range c.sources { rowBuf := &RowBuffer{rows: srcRows} sources = append(sources, rowBuf) } src, err := makeOrderedSync(c.ordering, sources) if err != nil { t.Fatal(err) } var retRows sqlbase.EncDatumRows for { row, err := src.NextRow() if err != nil { t.Fatal(err) } if row == nil { break } retRows = append(retRows, row) } expStr := c.expected.String() retStr := retRows.String() if expStr != retStr { t.Errorf("invalid results for case %d; expected:\n %s\ngot:\n %s", testIdx, expStr, retStr) } } }
// RowIdxFn is a GenValueFn that returns the row number as a DInt func RowIdxFn(row int) parser.Datum { return parser.NewDInt(parser.DInt(row)) }
// RowModuloFn creates a GenValueFn that returns the row number modulo a given // value as a DInt func RowModuloFn(modulo int) GenValueFn { return func(row int) parser.Datum { return parser.NewDInt(parser.DInt(row % modulo)) } }
func TestEncDatumRowCompare(t *testing.T) { v := [5]EncDatum{} for i := range v { v[i].SetDatum(ColumnType_INT, parser.NewDInt(parser.DInt(i))) } asc := encoding.Ascending desc := encoding.Descending testCases := []struct { row1, row2 EncDatumRow ord ColumnOrdering cmp int }{ { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{}, cmp: 0, }, { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{{1, desc}}, cmp: 0, }, { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{{0, asc}, {1, desc}}, cmp: 0, }, { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{{2, asc}}, cmp: -1, }, { row1: EncDatumRow{v[0], v[1], v[3]}, row2: EncDatumRow{v[0], v[1], v[2]}, ord: ColumnOrdering{{2, asc}}, cmp: 1, }, { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{{2, asc}, {0, asc}, {1, asc}}, cmp: -1, }, { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{{0, asc}, {2, desc}}, cmp: 1, }, { row1: EncDatumRow{v[0], v[1], v[2]}, row2: EncDatumRow{v[0], v[1], v[3]}, ord: ColumnOrdering{{1, desc}, {0, asc}, {2, desc}}, cmp: 1, }, { row1: EncDatumRow{v[2], v[3]}, row2: EncDatumRow{v[1], v[3], v[0]}, ord: ColumnOrdering{{0, asc}}, cmp: 1, }, { row1: EncDatumRow{v[2], v[3]}, row2: EncDatumRow{v[1], v[3], v[0]}, ord: ColumnOrdering{{1, desc}, {0, asc}}, cmp: 1, }, { row1: EncDatumRow{v[2], v[3]}, row2: EncDatumRow{v[1], v[3], v[0]}, ord: ColumnOrdering{{1, asc}, {0, asc}}, cmp: 1, }, { row1: EncDatumRow{v[2], v[3]}, row2: EncDatumRow{v[1], v[3], v[0]}, ord: ColumnOrdering{{1, asc}, {0, desc}}, cmp: -1, }, { row1: EncDatumRow{v[2], v[3]}, row2: EncDatumRow{v[1], v[3], v[0]}, ord: ColumnOrdering{{0, desc}, {1, asc}}, cmp: -1, }, } a := &DatumAlloc{} for _, c := range testCases { cmp, err := c.row1.Compare(a, c.ord, c.row2) if err != nil { t.Error(err) } else if cmp != c.cmp { t.Errorf("%s cmp %s ordering %v got %d, expected %d", c.row1, c.row2, c.ord, cmp, c.cmp) } } }