// writeTableDesc implements the SchemaAccessor interface.
func (p *planner) writeTableDesc(tableDesc *sqlbase.TableDescriptor) error {
if isVirtualDescriptor(tableDesc) {
panic(fmt.Sprintf("Virtual Descriptors cannot be stored, found: %v", tableDesc))
}
return p.txn.Put(sqlbase.MakeDescMetadataKey(tableDesc.GetID()),
sqlbase.WrapDescriptor(tableDesc))
}
func makeFKDeleteHelper(
txn *client.Txn, table sqlbase.TableDescriptor, otherTables tableLookupsByID, colMap map[sqlbase.ColumnID]int,
) (fkDeleteHelper, error) {
var fks fkDeleteHelper
for _, idx := range table.AllNonDropIndexes() {
for _, ref := range idx.ReferencedBy {
if otherTables[ref.Table].isAdding {
// We can assume that a table being added but not yet public is empty,
// and thus does not need to be checked for FK violations.
continue
}
fk, err := makeBaseFKHelper(txn, otherTables, idx, ref, colMap)
if err == errSkipUnsedFK {
continue
}
if err != nil {
return fks, err
}
if fks == nil {
fks = make(fkDeleteHelper)
}
fks[idx.ID] = append(fks[idx.ID], fk)
}
}
return fks, nil
}
func (p *planner) processColumns(tableDesc *sqlbase.TableDescriptor,
node parser.QualifiedNames) ([]sqlbase.ColumnDescriptor, error) {
if node == nil {
// VisibleColumns is used here to prevent INSERT INTO <table> VALUES (...)
// (as opposed to INSERT INTO <table> (...) VALUES (...)) from writing
// hidden columns. At present, the only hidden column is the implicit rowid
// primary key column.
return tableDesc.VisibleColumns(), nil
}
cols := make([]sqlbase.ColumnDescriptor, len(node))
colIDSet := make(map[sqlbase.ColumnID]struct{}, len(node))
for i, n := range node {
// TODO(pmattis): If the name is qualified, verify the table name matches
// tableDesc.Name.
if err := n.NormalizeColumnName(); err != nil {
return nil, err
}
col, err := tableDesc.FindActiveColumnByName(n.Column())
if err != nil {
return nil, err
}
if _, ok := colIDSet[col.ID]; ok {
return nil, fmt.Errorf("multiple assignments to same column \"%s\"", n.Column())
}
colIDSet[col.ID] = struct{}{}
cols[i] = col
}
return cols, nil
}
// getKeysForTableDescriptor retrieves the KV keys corresponding
// to the zone, name and descriptor of a table.
func getKeysForTableDescriptor(
tableDesc *sqlbase.TableDescriptor,
) (zoneKey roachpb.Key, nameKey roachpb.Key, descKey roachpb.Key) {
zoneKey = sqlbase.MakeZoneKey(tableDesc.ID)
nameKey = sqlbase.MakeNameMetadataKey(tableDesc.ParentID, tableDesc.GetName())
descKey = sqlbase.MakeDescMetadataKey(tableDesc.ID)
return
}
// makeRowDeleter creates a rowDeleter for the given table.
//
// The returned rowDeleter contains a fetchCols field that defines the
// expectation of which values are passed as values to deleteRow. Any column
// passed in requestedCols will be included in fetchCols.
func makeRowDeleter(
txn *client.Txn,
tableDesc *sqlbase.TableDescriptor,
fkTables TablesByID,
requestedCols []sqlbase.ColumnDescriptor,
checkFKs bool,
) (rowDeleter, error) {
indexes := tableDesc.Indexes
for _, m := range tableDesc.Mutations {
if index := m.GetIndex(); index != nil {
indexes = append(indexes, *index)
}
}
fetchCols := requestedCols[:len(requestedCols):len(requestedCols)]
fetchColIDtoRowIndex := colIDtoRowIndexFromCols(fetchCols)
maybeAddCol := func(colID sqlbase.ColumnID) error {
if _, ok := fetchColIDtoRowIndex[colID]; !ok {
col, err := tableDesc.FindColumnByID(colID)
if err != nil {
return err
}
fetchColIDtoRowIndex[col.ID] = len(fetchCols)
fetchCols = append(fetchCols, *col)
}
return nil
}
for _, colID := range tableDesc.PrimaryIndex.ColumnIDs {
if err := maybeAddCol(colID); err != nil {
return rowDeleter{}, err
}
}
for _, index := range indexes {
for _, colID := range index.ColumnIDs {
if err := maybeAddCol(colID); err != nil {
return rowDeleter{}, err
}
}
}
rd := rowDeleter{
helper: rowHelper{tableDesc: tableDesc, indexes: indexes},
fetchCols: fetchCols,
fetchColIDtoRowIndex: fetchColIDtoRowIndex,
}
if checkFKs {
var err error
if rd.fks, err = makeFKDeleteHelper(txn, *tableDesc, fkTables, fetchColIDtoRowIndex); err != nil {
return rowDeleter{}, nil
}
}
return rd, nil
}
func filterTableState(tableDesc *sqlbase.TableDescriptor) error {
switch {
case tableDesc.Deleted():
return errTableDeleted
case tableDesc.Adding():
return errTableAdding
case tableDesc.State != sqlbase.TableDescriptor_PUBLIC:
return errors.Errorf("table in unknown state: %s", tableDesc.State.String())
}
return nil
}
func makeColIDtoRowIndex(row planNode, desc *sqlbase.TableDescriptor) (
map[sqlbase.ColumnID]int, error,
) {
columns := row.Columns()
colIDtoRowIndex := make(map[sqlbase.ColumnID]int, len(columns))
for i, column := range columns {
col, err := desc.FindActiveColumnByName(column.Name)
if err != nil {
return nil, err
}
colIDtoRowIndex[col.ID] = i
}
return colIDtoRowIndex, nil
}
// deleteIndexMutationsWithReversedColumns deletes index mutations with a
// different mutationID than the schema changer and a reference to one of the
// reversed columns.
func (sc *SchemaChanger) deleteIndexMutationsWithReversedColumns(
desc *sqlbase.TableDescriptor, columns map[string]struct{},
) {
newMutations := make([]sqlbase.DescriptorMutation, 0, len(desc.Mutations))
for _, mutation := range desc.Mutations {
if mutation.MutationID != sc.mutationID {
if idx := mutation.GetIndex(); idx != nil {
deleteMutation := false
for _, name := range idx.ColumnNames {
if _, ok := columns[name]; ok {
// Such an index mutation has to be with direction ADD and
// in the DELETE_ONLY state. Live indexes referencing live
// columns cannot be deleted and thus never have direction
// DROP. All mutations with the ADD direction start off in
// the DELETE_ONLY state.
if mutation.Direction != sqlbase.DescriptorMutation_ADD ||
mutation.State != sqlbase.DescriptorMutation_DELETE_ONLY {
panic(fmt.Sprintf("mutation in bad state: %+v", mutation))
}
log.Warningf(context.TODO(), "delete schema change mutation: %+v", mutation)
deleteMutation = true
break
}
}
if deleteMutation {
continue
}
}
}
newMutations = append(newMutations, mutation)
}
// Reset mutations.
desc.Mutations = newMutations
}
func convertBatchError(tableDesc *sqlbase.TableDescriptor, b *client.Batch) error {
origPErr := b.MustPErr()
if origPErr.Index == nil {
return origPErr.GoError()
}
index := origPErr.Index.Index
if index >= int32(len(b.Results)) {
panic(fmt.Sprintf("index %d outside of results: %+v", index, b.Results))
}
result := b.Results[index]
var alloc sqlbase.DatumAlloc
if _, ok := origPErr.GetDetail().(*roachpb.ConditionFailedError); ok {
for _, row := range result.Rows {
// TODO(dan): There's too much internal knowledge of the sql table
// encoding here (and this callsite is the only reason
// DecodeIndexKeyPrefix is exported). Refactor this bit out.
indexID, key, err := sqlbase.DecodeIndexKeyPrefix(&alloc, tableDesc, row.Key)
if err != nil {
return err
}
index, err := tableDesc.FindIndexByID(indexID)
if err != nil {
return err
}
valTypes, err := sqlbase.MakeKeyVals(tableDesc, index.ColumnIDs)
if err != nil {
return err
}
dirs := make([]encoding.Direction, 0, len(index.ColumnIDs))
for _, dir := range index.ColumnDirections {
convertedDir, err := dir.ToEncodingDirection()
if err != nil {
return err
}
dirs = append(dirs, convertedDir)
}
vals := make([]parser.Datum, len(valTypes))
if _, err := sqlbase.DecodeKeyVals(&alloc, valTypes, vals, dirs, key); err != nil {
return err
}
return sqlbase.NewUniquenessConstraintViolationError(index, vals)
}
}
return origPErr.GoError()
}
func makeFKInsertHelper(
txn *client.Txn, table *sqlbase.TableDescriptor, otherTables TablesByID, colMap map[sqlbase.ColumnID]int,
) (fkInsertHelper, error) {
var fks fkInsertHelper
for _, idx := range table.AllNonDropIndexes() {
if idx.ForeignKey != nil {
fk, err := makeBaseFKHelper(txn, otherTables, idx, idx.ForeignKey, colMap)
if err != nil {
return fks, err
}
if fks == nil {
fks = make(fkInsertHelper)
}
fks[idx.ID] = append(fks[idx.ID], fk)
}
}
return fks, nil
}
func convertBackfillError(tableDesc *sqlbase.TableDescriptor, b *client.Batch) error {
// A backfill on a new schema element has failed and the batch contains
// information useful in printing a sensible error. However
// convertBatchError() will only work correctly if the schema elements are
// "live" in the tableDesc. Apply the mutations belonging to the same
// mutationID to make all the mutations live in tableDesc. Note: this
// tableDesc is not written to the k:v store.
mutationID := tableDesc.Mutations[0].MutationID
for _, mutation := range tableDesc.Mutations {
if mutation.MutationID != mutationID {
// Mutations are applied in a FIFO order. Only apply the first set
// of mutations if they have the mutation ID we're looking for.
break
}
tableDesc.MakeMutationComplete(mutation)
}
return convertBatchError(tableDesc, b)
}
func makeFKDeleteHelper(
txn *client.Txn, table *sqlbase.TableDescriptor, otherTables TablesByID, colMap map[sqlbase.ColumnID]int,
) (fkDeleteHelper, error) {
var fks fkDeleteHelper
for _, idx := range table.AllNonDropIndexes() {
for _, ref := range idx.ReferencedBy {
fk, err := makeBaseFKHelper(txn, otherTables, idx, ref, colMap)
if err != nil {
return fks, err
}
if fks == nil {
fks = make(fkDeleteHelper)
}
fks[idx.ID] = append(fks[idx.ID], fk)
}
}
return fks, nil
}
func convertBatchError(tableDesc *sqlbase.TableDescriptor, b client.Batch, origPErr *roachpb.Error) error {
if origPErr.Index == nil {
return origPErr.GoError()
}
index := origPErr.Index.Index
if index >= int32(len(b.Results)) {
panic(fmt.Sprintf("index %d outside of results: %+v", index, b.Results))
}
result := b.Results[index]
var alloc sqlbase.DatumAlloc
if _, ok := origPErr.GetDetail().(*roachpb.ConditionFailedError); ok {
for _, row := range result.Rows {
indexID, key, err := sqlbase.DecodeIndexKeyPrefix(tableDesc, row.Key)
if err != nil {
return err
}
index, err := tableDesc.FindIndexByID(indexID)
if err != nil {
return err
}
valTypes, err := sqlbase.MakeKeyVals(tableDesc, index.ColumnIDs)
if err != nil {
return err
}
dirs := make([]encoding.Direction, 0, len(index.ColumnIDs))
for _, dir := range index.ColumnDirections {
convertedDir, err := dir.ToEncodingDirection()
if err != nil {
return err
}
dirs = append(dirs, convertedDir)
}
vals := make([]parser.Datum, len(valTypes))
if _, err := sqlbase.DecodeKeyVals(&alloc, valTypes, vals, dirs, key); err != nil {
return err
}
return &errUniquenessConstraintViolation{index: index, vals: vals}
}
}
return origPErr.GoError()
}
func makeColIDtoRowIndex(row planNode, desc *sqlbase.TableDescriptor) (
map[sqlbase.ColumnID]int, error,
) {
columns := row.Columns()
colIDtoRowIndex := make(map[sqlbase.ColumnID]int, len(columns))
for i, column := range columns {
s, idx, err := desc.FindColumnByName(column.Name)
if err != nil {
return nil, err
}
switch s {
case sqlbase.DescriptorActive:
colIDtoRowIndex[desc.Columns[idx].ID] = i
case sqlbase.DescriptorIncomplete:
colIDtoRowIndex[desc.Mutations[idx].GetColumn().ID] = i
default:
panic("unreachable")
}
}
return colIDtoRowIndex, nil
}
// tablesNeededForFKs calculates the IDs of the additional TableDescriptors that
// will be needed for FK checking delete and/or insert operations on `table`.
//
// NB: the returned map's values are *not* set -- higher level calling code, eg
// in planner, should fill the map's values by acquiring leases. This function
// is essentially just returning a slice of IDs, but the empty map can be filled
// in place and reused, avoiding a second allocation.
func tablesNeededForFKs(table sqlbase.TableDescriptor, usage FKCheck) tableLookupsByID {
var ret tableLookupsByID
for _, idx := range table.AllNonDropIndexes() {
if usage != CheckDeletes && idx.ForeignKey.IsSet() {
if ret == nil {
ret = make(tableLookupsByID)
}
ret[idx.ForeignKey.Table] = tableLookup{}
}
if usage != CheckInserts {
for _, idx := range table.AllNonDropIndexes() {
for _, ref := range idx.ReferencedBy {
if ret == nil {
ret = make(tableLookupsByID)
}
ret[ref.Table] = tableLookup{}
}
}
}
}
return ret
}
// makeRowDeleter creates a rowDeleter for the given table.
//
// The returned rowDeleter contains a fetchCols field that defines the
// expectation of which values are passed as values to deleteRow.
func makeRowDeleter(
tableDesc *sqlbase.TableDescriptor,
) (rowDeleter, error) {
// TODO(dan): makeRowDeleter should take a param for the sql rows needed for
// returningHelper, etc.
requestedCols := tableDesc.Columns
indexes := tableDesc.Indexes
for _, m := range tableDesc.Mutations {
if index := m.GetIndex(); index != nil {
indexes = append(indexes, *index)
}
}
fetchCols := requestedCols[:len(requestedCols):len(requestedCols)]
fetchColIDtoRowIndex := colIDtoRowIndexFromCols(fetchCols)
for _, index := range indexes {
for _, colID := range index.ColumnIDs {
if _, ok := fetchColIDtoRowIndex[colID]; !ok {
// TODO(dan): What about non-active columns?
col, err := tableDesc.FindActiveColumnByID(colID)
if err != nil {
return rowDeleter{}, err
}
fetchColIDtoRowIndex[colID] = len(fetchCols)
fetchCols = append(fetchCols, *col)
}
}
}
rd := rowDeleter{
helper: rowHelper{tableDesc: tableDesc, indexes: indexes},
fetchCols: fetchCols,
fetchColIDtoRowIndex: fetchColIDtoRowIndex,
}
return rd, nil
}
func restoreTable(
ctx context.Context,
sst engine.RocksDBSstFileReader,
txn *client.Txn,
table *sqlbase.TableDescriptor,
overwrite bool,
) error {
log.Infof(ctx, "Restoring Table %q", table.Name)
tableStartKey := roachpb.Key(sqlbase.MakeIndexKeyPrefix(table, table.PrimaryIndex.ID))
tableEndKey := tableStartKey.PrefixEnd()
existingDesc, err := txn.Get(sqlbase.MakeDescMetadataKey(table.GetID()))
if err != nil {
return err
}
existingData, err := txn.Scan(tableStartKey, tableEndKey, 1)
if err != nil {
return err
}
if existingDesc.Value != nil || len(existingData) > 0 {
if overwrite {
// We're about to Put the descriptor, so don't bother deleting it.
if err := txn.DelRange(tableStartKey, tableEndKey); err != nil {
return err
}
} else {
return errors.Errorf("table %q already exists", table.Name)
}
}
tableDescKey := sqlbase.MakeDescMetadataKey(table.GetID())
if err := txn.Put(tableDescKey, sqlbase.WrapDescriptor(table)); err != nil {
return err
}
return Import(ctx, sst, txn, engine.MVCCKey{Key: tableStartKey}, engine.MVCCKey{Key: tableEndKey})
}
func (p *planner) processColumns(tableDesc *sqlbase.TableDescriptor,
node parser.UnresolvedNames) ([]sqlbase.ColumnDescriptor, error) {
if node == nil {
// VisibleColumns is used here to prevent INSERT INTO <table> VALUES (...)
// (as opposed to INSERT INTO <table> (...) VALUES (...)) from writing
// hidden columns. At present, the only hidden column is the implicit rowid
// primary key column.
return tableDesc.VisibleColumns(), nil
}
cols := make([]sqlbase.ColumnDescriptor, len(node))
colIDSet := make(map[sqlbase.ColumnID]struct{}, len(node))
for i, n := range node {
c, err := n.NormalizeUnqualifiedColumnItem()
if err != nil {
return nil, err
}
if len(c.Selector) > 0 {
return nil, util.UnimplementedWithIssueErrorf(8318, "compound types not supported yet: %q", n)
}
col, err := tableDesc.FindActiveColumnByName(c.ColumnName)
if err != nil {
return nil, err
}
if _, ok := colIDSet[col.ID]; ok {
return nil, fmt.Errorf("multiple assignments to the same column %q", n)
}
colIDSet[col.ID] = struct{}{}
cols[i] = col
}
return cols, nil
}
// makeRowUpdater creates a rowUpdater for the given table.
//
// updateCols are the columns being updated and correspond to the updateValues
// that will be passed to updateRow.
//
// The returned rowUpdater contains a fetchCols field that defines the
// expectation of which values are passed as oldValues to updateRow. Any column
// passed in requestedCols will be included in fetchCols.
func makeRowUpdater(
txn *client.Txn,
tableDesc *sqlbase.TableDescriptor,
fkTables TablesByID,
updateCols []sqlbase.ColumnDescriptor,
requestedCols []sqlbase.ColumnDescriptor,
updateType rowUpdaterType,
) (rowUpdater, error) {
updateColIDtoRowIndex := colIDtoRowIndexFromCols(updateCols)
primaryIndexCols := make(map[sqlbase.ColumnID]struct{}, len(tableDesc.PrimaryIndex.ColumnIDs))
for _, colID := range tableDesc.PrimaryIndex.ColumnIDs {
primaryIndexCols[colID] = struct{}{}
}
var primaryKeyColChange bool
for _, c := range updateCols {
if _, ok := primaryIndexCols[c.ID]; ok {
primaryKeyColChange = true
break
}
}
// Secondary indexes needing updating.
needsUpdate := func(index sqlbase.IndexDescriptor) bool {
if updateType == rowUpdaterOnlyColumns {
// Only update columns.
return false
}
// If the primary key changed, we need to update all of them.
if primaryKeyColChange {
return true
}
for _, id := range index.ColumnIDs {
if _, ok := updateColIDtoRowIndex[id]; ok {
return true
}
}
return false
}
indexes := make([]sqlbase.IndexDescriptor, 0, len(tableDesc.Indexes)+len(tableDesc.Mutations))
for _, index := range tableDesc.Indexes {
if needsUpdate(index) {
indexes = append(indexes, index)
}
}
var deleteOnlyIndex map[int]struct{}
for _, m := range tableDesc.Mutations {
if index := m.GetIndex(); index != nil {
if needsUpdate(*index) {
indexes = append(indexes, *index)
switch m.State {
case sqlbase.DescriptorMutation_DELETE_ONLY:
if deleteOnlyIndex == nil {
// Allocate at most once.
deleteOnlyIndex = make(map[int]struct{}, len(tableDesc.Mutations))
}
deleteOnlyIndex[len(indexes)-1] = struct{}{}
case sqlbase.DescriptorMutation_WRITE_ONLY:
}
}
}
}
ru := rowUpdater{
helper: rowHelper{tableDesc: tableDesc, indexes: indexes},
updateCols: updateCols,
updateColIDtoRowIndex: updateColIDtoRowIndex,
deleteOnlyIndex: deleteOnlyIndex,
primaryKeyColChange: primaryKeyColChange,
marshalled: make([]roachpb.Value, len(updateCols)),
newValues: make([]parser.Datum, len(tableDesc.Columns)+len(tableDesc.Mutations)),
}
if primaryKeyColChange {
// These fields are only used when the primary key is changing.
var err error
// When changing the primary key, we delete the old values and reinsert
// them, so request them all.
if ru.rd, err = makeRowDeleter(txn, tableDesc, fkTables, tableDesc.Columns, skipFKs); err != nil {
return rowUpdater{}, err
}
ru.fetchCols = ru.rd.fetchCols
ru.fetchColIDtoRowIndex = colIDtoRowIndexFromCols(ru.fetchCols)
if ru.ri, err = makeRowInserter(txn, tableDesc, fkTables, tableDesc.Columns, skipFKs); err != nil {
return rowUpdater{}, err
}
} else {
ru.fetchCols = requestedCols[:len(requestedCols):len(requestedCols)]
ru.fetchColIDtoRowIndex = colIDtoRowIndexFromCols(ru.fetchCols)
maybeAddCol := func(colID sqlbase.ColumnID) error {
if _, ok := ru.fetchColIDtoRowIndex[colID]; !ok {
col, err := tableDesc.FindColumnByID(colID)
if err != nil {
return err
}
ru.fetchColIDtoRowIndex[col.ID] = len(ru.fetchCols)
ru.fetchCols = append(ru.fetchCols, *col)
}
return nil
}
for _, colID := range tableDesc.PrimaryIndex.ColumnIDs {
if err := maybeAddCol(colID); err != nil {
return rowUpdater{}, err
}
}
for _, fam := range tableDesc.Families {
familyBeingUpdated := false
for _, colID := range fam.ColumnIDs {
if _, ok := ru.updateColIDtoRowIndex[colID]; ok {
familyBeingUpdated = true
break
}
}
if familyBeingUpdated {
for _, colID := range fam.ColumnIDs {
if err := maybeAddCol(colID); err != nil {
return rowUpdater{}, err
}
}
}
}
for _, index := range indexes {
for _, colID := range index.ColumnIDs {
if err := maybeAddCol(colID); err != nil {
return rowUpdater{}, err
}
}
}
}
var err error
if ru.fks, err = makeFKUpdateHelper(txn, *tableDesc, fkTables, ru.fetchColIDtoRowIndex); err != nil {
return rowUpdater{}, err
}
return ru, nil
}
// writeTableDesc implements the SchemaAccessor interface.
func (p *planner) writeTableDesc(tableDesc *sqlbase.TableDescriptor) error {
return p.txn.Put(sqlbase.MakeDescMetadataKey(tableDesc.GetID()),
sqlbase.WrapDescriptor(tableDesc))
}