// TestStoreRangeSplitInsideRow verifies an attempt to split a range inside of
// a table row will cause a split at a boundary between rows.
func TestStoreRangeSplitInsideRow(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
store, stopper, _ := createTestStore(t)
defer stopper.Stop()
// Manually create some the column keys corresponding to the table:
//
// CREATE TABLE t (id STRING PRIMARY KEY, col1 INT, col2 INT)
tableKey := keys.MakeTablePrefix(keys.MaxReservedDescID + 1)
rowKey := roachpb.Key(encoding.EncodeVarintAscending(append([]byte(nil), tableKey...), 1))
rowKey = encoding.EncodeStringAscending(encoding.EncodeVarintAscending(rowKey, 1), "a")
col1Key := keys.MakeColumnKey(append([]byte(nil), rowKey...), 1)
col2Key := keys.MakeColumnKey(append([]byte(nil), rowKey...), 2)
// We don't care about the value, so just store any old thing.
if pErr := store.DB().Put(col1Key, "column 1"); pErr != nil {
t.Fatal(pErr)
}
if pErr := store.DB().Put(col2Key, "column 2"); pErr != nil {
t.Fatal(pErr)
}
// Split between col1Key and col2Key by splitting before col2Key.
args := adminSplitArgs(col2Key, col2Key)
_, pErr := client.SendWrapped(rg1(store), nil, &args)
if pErr != nil {
t.Fatalf("%s: split unexpected error: %s", col1Key, pErr)
}
rng1 := store.LookupReplica(col1Key, nil)
rng2 := store.LookupReplica(col2Key, nil)
// Verify the two columns are still on the same range.
if !reflect.DeepEqual(rng1, rng2) {
t.Fatalf("%s: ranges differ: %+v vs %+v", roachpb.Key(col1Key), rng1, rng2)
}
// Verify we split on a row key.
if startKey := rng1.Desc().StartKey; !startKey.Equal(rowKey) {
t.Fatalf("%s: expected split on %s, but found %s",
roachpb.Key(col1Key), roachpb.Key(rowKey), startKey)
}
// Verify the previous range was split on a row key.
rng3 := store.LookupReplica(tableKey, nil)
if endKey := rng3.Desc().EndKey; !endKey.Equal(rowKey) {
t.Fatalf("%s: expected split on %s, but found %s",
roachpb.Key(col1Key), roachpb.Key(rowKey), endKey)
}
}
// MakeNameMetadataKey returns the key for the name. Pass name == "" in order
// to generate the prefix key to use to scan over all of the names for the
// specified parentID.
func MakeNameMetadataKey(parentID ID, name string) roachpb.Key {
name = NormalizeName(name)
k := keys.MakeTablePrefix(uint32(namespaceTable.ID))
k = encoding.EncodeUvarintAscending(k, uint64(namespaceTable.PrimaryIndex.ID))
k = encoding.EncodeUvarintAscending(k, uint64(parentID))
if name != "" {
k = encoding.EncodeBytesAscending(k, []byte(name))
k = keys.MakeColumnKey(k, uint32(namespaceTable.Columns[2].ID))
}
return k
}
// MakeDescMetadataKey returns the key for the descriptor.
func MakeDescMetadataKey(descID ID) roachpb.Key {
k := keys.MakeTablePrefix(uint32(descriptorTable.ID))
k = encoding.EncodeUvarintAscending(k, uint64(descriptorTable.PrimaryIndex.ID))
k = encoding.EncodeUvarintAscending(k, uint64(descID))
return keys.MakeColumnKey(k, uint32(descriptorTable.Columns[1].ID))
}
// MakeZoneKey returns the key for 'id's entry in the system.zones table.
func MakeZoneKey(id ID) roachpb.Key {
k := keys.MakeTablePrefix(uint32(zonesTable.ID))
k = encoding.EncodeUvarintAscending(k, uint64(zonesTable.PrimaryIndex.ID))
k = encoding.EncodeUvarintAscending(k, uint64(id))
return keys.MakeColumnKey(k, uint32(zonesTable.Columns[1].ID))
}
// insertRow adds to the batch the kv operations necessary to insert a table row
// with the given values.
func (ri *rowInserter) insertRow(b *client.Batch, values []parser.Datum) error {
if len(values) != len(ri.insertCols) {
return util.Errorf("got %d values but expected %d", len(values), len(ri.insertCols))
}
// Encode the values to the expected column type. This needs to
// happen before index encoding because certain datum types (i.e. tuple)
// cannot be used as index values.
for i, val := range values {
// Make sure the value can be written to the column before proceeding.
var err error
if ri.marshalled[i], err = sqlbase.MarshalColumnValue(ri.insertCols[i], val); err != nil {
return err
}
}
primaryIndexKey, secondaryIndexEntries, err := ri.helper.encodeIndexes(ri.insertColIDtoRowIndex, values)
if err != nil {
return err
}
// Write the row sentinel. We want to write the sentinel first in case
// we are trying to insert a duplicate primary key: if we write the
// secondary indexes first, we may get an error that looks like a
// uniqueness violation on a non-unique index.
ri.key = keys.MakeNonColumnKey(primaryIndexKey)
if log.V(2) {
log.Infof("CPut %s -> NULL", ri.key)
}
// Each sentinel value needs a distinct RawBytes field as the computed
// checksum includes the key the value is associated with.
ri.sentinelValue.SetBytes([]byte{})
b.CPut(&ri.key, &ri.sentinelValue, nil)
ri.key = nil
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("CPut %s -> %v", secondaryIndexEntry.Key, secondaryIndexEntry.Value)
}
ri.key = secondaryIndexEntry.Key
b.CPut(&ri.key, secondaryIndexEntry.Value, nil)
}
ri.key = nil
// Write the row columns.
for i, val := range values {
col := ri.insertCols[i]
if ri.helper.columnInPK(col.ID) {
// Skip primary key columns as their values are encoded in the row
// sentinel key which is guaranteed to exist for as long as the row
// exists.
continue
}
if ri.marshalled[i].RawBytes != nil {
// We only output non-NULL values. Non-existent column keys are
// considered NULL during scanning and the row sentinel ensures we know
// the row exists.
ri.key = keys.MakeColumnKey(primaryIndexKey, uint32(col.ID))
if log.V(2) {
log.Infof("CPut %s -> %v", ri.key, val)
}
b.CPut(&ri.key, &ri.marshalled[i], nil)
ri.key = nil
}
}
return nil
}
// updateRow adds to the batch the kv operations necessary to update a table row
// with the given values.
//
// The row corresponding to oldValues is updated with the ones in updateValues.
// Note that updateValues only contains the ones that are changing.
//
// The return value is only good until the next call to UpdateRow.
func (ru *rowUpdater) updateRow(
b *client.Batch,
oldValues []parser.Datum,
updateValues []parser.Datum,
) ([]parser.Datum, error) {
if len(oldValues) != len(ru.fetchCols) {
return nil, util.Errorf("got %d values but expected %d", len(oldValues), len(ru.fetchCols))
}
if len(updateValues) != len(ru.updateCols) {
return nil, util.Errorf("got %d values but expected %d", len(updateValues), len(ru.updateCols))
}
primaryIndexKey, secondaryIndexEntries, err := ru.helper.encodeIndexes(ru.fetchColIDtoRowIndex, oldValues)
if err != nil {
return nil, err
}
// Check that the new value types match the column types. This needs to
// happen before index encoding because certain datum types (i.e. tuple)
// cannot be used as index values.
for i, val := range updateValues {
if ru.marshalled[i], err = sqlbase.MarshalColumnValue(ru.updateCols[i], val); err != nil {
return nil, err
}
}
// Update the row values.
copy(ru.newValues, oldValues)
for i, updateCol := range ru.updateCols {
ru.newValues[ru.fetchColIDtoRowIndex[updateCol.ID]] = updateValues[i]
}
newPrimaryIndexKey := primaryIndexKey
rowPrimaryKeyChanged := false
var newSecondaryIndexEntries []sqlbase.IndexEntry
if ru.primaryKeyColChange {
newPrimaryIndexKey, newSecondaryIndexEntries, err = ru.helper.encodeIndexes(ru.fetchColIDtoRowIndex, ru.newValues)
if err != nil {
return nil, err
}
rowPrimaryKeyChanged = !bytes.Equal(primaryIndexKey, newPrimaryIndexKey)
} else {
newSecondaryIndexEntries, err = sqlbase.EncodeSecondaryIndexes(
ru.helper.tableDesc.ID, ru.helper.indexes, ru.fetchColIDtoRowIndex, ru.newValues)
if err != nil {
return nil, err
}
}
if rowPrimaryKeyChanged {
err := ru.rd.deleteRow(b, oldValues)
if err != nil {
return nil, err
}
err = ru.ri.insertRow(b, ru.newValues)
return ru.newValues, err
}
// Update secondary indexes.
for i, newSecondaryIndexEntry := range newSecondaryIndexEntries {
secondaryIndexEntry := secondaryIndexEntries[i]
secondaryKeyChanged := !bytes.Equal(newSecondaryIndexEntry.Key, secondaryIndexEntry.Key)
if secondaryKeyChanged {
if log.V(2) {
log.Infof("Del %s", secondaryIndexEntry.Key)
}
b.Del(secondaryIndexEntry.Key)
// Do not update Indexes in the DELETE_ONLY state.
if _, ok := ru.deleteOnlyIndex[i]; !ok {
if log.V(2) {
log.Infof("CPut %s -> %v", newSecondaryIndexEntry.Key, newSecondaryIndexEntry.Value)
}
b.CPut(newSecondaryIndexEntry.Key, newSecondaryIndexEntry.Value, nil)
}
}
}
// Add the new values.
for i, val := range updateValues {
col := ru.updateCols[i]
if ru.helper.columnInPK(col.ID) {
// Skip primary key columns as their values are encoded in the row
// sentinel key which is guaranteed to exist for as long as the row
// exists.
continue
}
ru.key = keys.MakeColumnKey(newPrimaryIndexKey, uint32(col.ID))
if ru.marshalled[i].RawBytes != nil {
// We only output non-NULL values. Non-existent column keys are
// considered NULL during scanning and the row sentinel ensures we know
// the row exists.
if log.V(2) {
log.Infof("Put %s -> %v", ru.key, val)
}
b.Put(&ru.key, &ru.marshalled[i])
} else {
// The column might have already existed but is being set to NULL, so
// delete it.
if log.V(2) {
log.Infof("Del %s", ru.key)
}
b.Del(&ru.key)
}
ru.key = nil
}
return ru.newValues, nil
}
func (sc *SchemaChanger) truncateAndBackfillColumnsChunk(
added []sqlbase.ColumnDescriptor,
dropped []sqlbase.ColumnDescriptor,
nonNullableColumn string,
defaultExprs []parser.TypedExpr,
evalCtx parser.EvalContext,
sp sqlbase.Span,
) (roachpb.Key, bool, error) {
var curSentinel roachpb.Key
done := false
err := sc.db.Txn(func(txn *client.Txn) error {
tableDesc, err := getTableDescFromID(txn, sc.tableID)
if err != nil {
return err
}
// Short circuit the backfill if the table has been deleted.
if tableDesc.Deleted {
done = true
return nil
}
// Run a scan across the table using the primary key. Running
// the scan and applying the changes in many transactions is
// fine because the schema change is in the correct state to
// handle intermediate OLTP commands which delete and add
// values during the scan.
b := &client.Batch{}
b.Scan(sp.Start, sp.End, ColumnTruncateAndBackfillChunkSize)
if err := txn.Run(b); err != nil {
return err
}
// Use a different batch to truncate/backfill columns.
writeBatch := &client.Batch{}
marshalled := make([]roachpb.Value, len(defaultExprs))
done = true
for _, result := range b.Results {
var sentinelKey roachpb.Key
for _, kv := range result.Rows {
// Still processing table.
done = false
if nonNullableColumn != "" {
return fmt.Errorf("column %s contains null values", nonNullableColumn)
}
if sentinelKey == nil || !bytes.HasPrefix(kv.Key, sentinelKey) {
// Sentinel keys have a 0 suffix indicating 0 bytes of
// column ID. Strip off that suffix to determine the
// prefix shared with the other keys for the row.
sentinelKey = sqlbase.StripColumnIDLength(kv.Key)
// Store away key for the next table row as the point from
// which to start from.
curSentinel = sentinelKey
// Delete the entire dropped columns. This used to use SQL
// UPDATE in the past to update the dropped column to
// NULL; but a column in the process of being dropped is
// placed in the table descriptor mutations, and a SQL
// UPDATE of a column in mutations will fail.
for _, columnDesc := range dropped {
// Delete the dropped column.
colKey := keys.MakeColumnKey(sentinelKey, uint32(columnDesc.ID))
if log.V(2) {
log.Infof("Del %s", colKey)
}
writeBatch.Del(colKey)
}
// Add the new columns and backfill the values.
for i, expr := range defaultExprs {
if expr == nil {
continue
}
col := added[i]
colKey := keys.MakeColumnKey(sentinelKey, uint32(col.ID))
d, err := expr.Eval(evalCtx)
if err != nil {
return err
}
marshalled[i], err = sqlbase.MarshalColumnValue(col, d)
if err != nil {
return err
}
if log.V(2) {
log.Infof("Put %s -> %v", colKey, d)
}
// Insert default value into the column. If this row
// was recently added the default value might have
// already been populated, because the
// ColumnDescriptor is in the WRITE_ONLY state.
// Reinserting the default value is not a big deal.
//
// Note: a column in the WRITE_ONLY state cannot be
// populated directly through SQL. A SQL INSERT cannot
// directly reference the column, and the INSERT
// populates the column with the default value.
writeBatch.Put(colKey, &marshalled[i])
}
}
}
}
if err := txn.Run(writeBatch); err != nil {
for _, r := range writeBatch.Results {
if r.PErr != nil {
return convertBackfillError(tableDesc, writeBatch, r.PErr)
}
}
return err
}
return nil
})
return curSentinel.PrefixEnd(), done, err
}
func (p *planner) backfillBatch(b *client.Batch, oldTableDesc *TableDescriptor, mutationID MutationID) *roachpb.Error {
var droppedColumnDescs []ColumnDescriptor
var droppedIndexDescs []IndexDescriptor
var newIndexDescs []IndexDescriptor
// Collect the elements that are part of the mutation.
for _, m := range oldTableDesc.Mutations {
if m.MutationID != mutationID {
// Mutations are applied in a FIFO order. Only apply the first set of
// mutations if they have the mutation ID we're looking for.
break
}
switch m.Direction {
case DescriptorMutation_ADD:
switch t := m.Descriptor_.(type) {
case *DescriptorMutation_Column:
// TODO(vivek): Add column to new columns and use it
// to fill in default values.
case *DescriptorMutation_Index:
newIndexDescs = append(newIndexDescs, *t.Index)
}
case DescriptorMutation_DROP:
switch t := m.Descriptor_.(type) {
case *DescriptorMutation_Column:
droppedColumnDescs = append(droppedColumnDescs, *t.Column)
case *DescriptorMutation_Index:
droppedIndexDescs = append(droppedIndexDescs, *t.Index)
}
}
}
// TODO(vivek): Break these backfill operations into chunks. All of them
// will fail on big tables (see #3274).
// Delete the entire dropped columns.
// This used to use SQL UPDATE in the past to update the dropped
// column to NULL; but a column in the process of being
// dropped is placed in the table descriptor mutations, and
// a SQL UPDATE of a column in mutations will fail.
if len(droppedColumnDescs) > 0 {
// Run a scan across the table using the primary key.
start := roachpb.Key(MakeIndexKeyPrefix(oldTableDesc.ID, oldTableDesc.PrimaryIndex.ID))
// Use a different batch to perform the scan.
batch := &client.Batch{}
batch.Scan(start, start.PrefixEnd(), 0)
if pErr := p.txn.Run(batch); pErr != nil {
return pErr
}
for _, result := range batch.Results {
var sentinelKey roachpb.Key
for _, kv := range result.Rows {
if sentinelKey == nil || !bytes.HasPrefix(kv.Key, sentinelKey) {
// Sentinel keys have a 0 suffix indicating 0 bytes of column
// ID. Strip off that suffix to determine the prefix shared with the
// other keys for the row.
sentinelKey = stripColumnIDLength(kv.Key)
for _, columnDesc := range droppedColumnDescs {
// Delete the dropped column.
colKey := keys.MakeColumnKey(sentinelKey, uint32(columnDesc.ID))
if log.V(2) {
log.Infof("Del %s", colKey)
}
b.Del(colKey)
}
}
}
}
}
for _, indexDescriptor := range droppedIndexDescs {
indexPrefix := MakeIndexKeyPrefix(oldTableDesc.ID, indexDescriptor.ID)
// Delete the index.
indexStartKey := roachpb.Key(indexPrefix)
indexEndKey := indexStartKey.PrefixEnd()
if log.V(2) {
log.Infof("DelRange %s - %s", indexStartKey, indexEndKey)
}
b.DelRange(indexStartKey, indexEndKey)
}
if len(newIndexDescs) > 0 {
// Get all the rows affected.
// TODO(vivek): Avoid going through Select.
// TODO(tamird): Support partial indexes?
// Use a scanNode with SELECT to pass in a TableDescriptor
// to the SELECT without needing to use a parser.QualifiedName,
// because we want to run schema changes from a gossip feed of
// table IDs.
scan := &scanNode{
planner: p,
txn: p.txn,
desc: oldTableDesc,
}
scan.initDescDefaults()
rows, pErr := p.selectIndex(&selectNode{}, scan, nil, false)
if pErr != nil {
return pErr
}
// Construct a map from column ID to the index the value appears at within a
// row.
colIDtoRowIndex, pErr := makeColIDtoRowIndex(rows, oldTableDesc)
if pErr != nil {
return pErr
}
for rows.Next() {
rowVals := rows.Values()
for _, newIndexDesc := range newIndexDescs {
secondaryIndexEntries, pErr := encodeSecondaryIndexes(
oldTableDesc.ID, []IndexDescriptor{newIndexDesc}, colIDtoRowIndex, rowVals)
if pErr != nil {
return pErr
}
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("CPut %s -> %v", secondaryIndexEntry.key,
secondaryIndexEntry.value)
}
b.CPut(secondaryIndexEntry.key, secondaryIndexEntry.value, nil)
}
}
}
return rows.PErr()
}
return nil
}
func (p *planner) backfillBatch(b *client.Batch, oldTableDesc, newTableDesc *TableDescriptor) error {
var droppedColumnDescs []ColumnDescriptor
var droppedIndexDescs []IndexDescriptor
var newIndexDescs []IndexDescriptor
for _, m := range oldTableDesc.Mutations {
switch m.Direction {
case DescriptorMutation_ADD:
switch t := m.Descriptor_.(type) {
case *DescriptorMutation_Column:
// TODO(vivek): Add column to new columns and use it
// to fill in default values.
case *DescriptorMutation_Index:
newIndexDescs = append(newIndexDescs, *t.Index)
}
case DescriptorMutation_DROP:
switch t := m.Descriptor_.(type) {
case *DescriptorMutation_Column:
droppedColumnDescs = append(droppedColumnDescs, *t.Column)
case *DescriptorMutation_Index:
droppedIndexDescs = append(droppedIndexDescs, *t.Index)
}
}
}
// TODO(vivek): Break these backfill operations into chunks. All of them
// will fail on big tables.
// Delete the entire dropped columns.
// This used to use SQL UPDATE in the past to update the dropped
// column to NULL; but a column in the process of being
// dropped is placed in the table descriptor mutations, and
// a SQL UPDATE of a column in mutations will fail.
if len(droppedColumnDescs) > 0 {
// Run a scan across the table using the primary key.
start := roachpb.Key(MakeIndexKeyPrefix(newTableDesc.ID, newTableDesc.PrimaryIndex.ID))
// Use a different batch to perform the scan.
batch := &client.Batch{}
batch.Scan(start, start.PrefixEnd(), 0)
if err := p.txn.Run(batch); err != nil {
return err
}
for _, result := range batch.Results {
var sentinelKey roachpb.Key
for _, kv := range result.Rows {
if sentinelKey == nil || !bytes.HasPrefix(kv.Key, sentinelKey) {
// Sentinel keys have a 0 suffix indicating 0 bytes of column
// ID. Strip off that suffix to determine the prefix shared with the
// other keys for the row.
sentinelKey = stripColumnIDLength(kv.Key)
for _, columnDesc := range droppedColumnDescs {
// Delete the dropped column.
colKey := keys.MakeColumnKey(sentinelKey, uint32(columnDesc.ID))
if log.V(2) {
log.Infof("Del %s", colKey)
}
b.Del(colKey)
}
}
}
}
}
for _, indexDescriptor := range droppedIndexDescs {
indexPrefix := MakeIndexKeyPrefix(newTableDesc.ID, indexDescriptor.ID)
// Delete the index.
indexStartKey := roachpb.Key(indexPrefix)
indexEndKey := indexStartKey.PrefixEnd()
if log.V(2) {
log.Infof("DelRange %s - %s", indexStartKey, indexEndKey)
}
b.DelRange(indexStartKey, indexEndKey)
}
if len(newIndexDescs) > 0 {
// Get all the rows affected.
// TODO(vivek): Avoid going through Select.
// TODO(tamird): Support partial indexes?
// Use a scanNode with SELECT to pass in a TableDescriptor
// to the SELECT without needing to use a parser.QualifiedName,
// because we want to run schema changes from a gossip feed of
// table IDs.
scan := &scanNode{
planner: p,
txn: p.txn,
desc: oldTableDesc,
}
scan.initDescDefaults()
rows, err := p.selectWithScan(scan, &parser.Select{Exprs: oldTableDesc.allColumnsSelector()})
if err != nil {
return err
}
// Construct a map from column ID to the index the value appears at within a
// row.
colIDtoRowIndex, err := makeColIDtoRowIndex(rows, oldTableDesc)
if err != nil {
return err
}
// TODO(tamird): This will fall down in production use. We need to do
// something better (see #2036). In particular, this implementation
// has the following problems:
// - Very large tables will generate an enormous batch here. This
// isn't really a problem in itself except that it will exacerbate
// the other issue:
// - Any non-quiescent table that this runs against will end up with
// an inconsistent index. This is because as inserts/updates continue
// to roll in behind this operation's read front, the written index
// will become incomplete/stale before it's written.
for rows.Next() {
rowVals := rows.Values()
for _, newIndexDesc := range newIndexDescs {
secondaryIndexEntries, err := encodeSecondaryIndexes(
oldTableDesc.ID, []IndexDescriptor{newIndexDesc}, colIDtoRowIndex, rowVals)
if err != nil {
return err
}
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("CPut %s -> %v", secondaryIndexEntry.key,
secondaryIndexEntry.value)
}
b.CPut(secondaryIndexEntry.key, secondaryIndexEntry.value, nil)
}
}
}
return rows.Err()
}
return nil
}
// Insert inserts rows into the database.
// Privileges: INSERT on table
// Notes: postgres requires INSERT. No "on duplicate key update" option.
// mysql requires INSERT. Also requires UPDATE on "ON DUPLICATE KEY UPDATE".
func (p *planner) Insert(n *parser.Insert, autoCommit bool) (planNode, *roachpb.Error) {
// TODO(marcb): We can't use the cached descriptor here because a recent
// update of the schema (e.g. the addition of an index) might not be
// reflected in the cached version (yet). Perhaps schema modification
// routines such as CREATE INDEX should not return until the schema change
// has been pushed everywhere.
tableDesc, pErr := p.getTableLease(n.Table)
if pErr != nil {
return nil, pErr
}
if err := p.checkPrivilege(&tableDesc, privilege.INSERT); err != nil {
return nil, roachpb.NewError(err)
}
var cols []ColumnDescriptor
// Determine which columns we're inserting into.
if n.DefaultValues() {
cols = tableDesc.Columns
} else {
var err error
if cols, err = p.processColumns(&tableDesc, n.Columns); err != nil {
return nil, roachpb.NewError(err)
}
}
// Number of columns expecting an input. This doesn't include the
// columns receiving a default value.
numInputColumns := len(cols)
// Construct a map from column ID to the index the value appears at within a
// row.
colIDtoRowIndex := map[ColumnID]int{}
for i, c := range cols {
colIDtoRowIndex[c.ID] = i
}
// Add the column if it has a DEFAULT expression.
addIfDefault := func(col ColumnDescriptor) {
if col.DefaultExpr != nil {
if _, ok := colIDtoRowIndex[col.ID]; !ok {
colIDtoRowIndex[col.ID] = len(cols)
cols = append(cols, col)
}
}
}
// Add any column that has a DEFAULT expression.
for _, col := range tableDesc.Columns {
addIfDefault(col)
}
// Also add any column in a mutation that is WRITE_ONLY and has
// a DEFAULT expression.
for _, m := range tableDesc.Mutations {
if m.State != DescriptorMutation_WRITE_ONLY {
continue
}
if col := m.GetColumn(); col != nil {
addIfDefault(*col)
}
}
// Verify we have at least the columns that are part of the primary key.
primaryKeyCols := map[ColumnID]struct{}{}
for i, id := range tableDesc.PrimaryIndex.ColumnIDs {
if _, ok := colIDtoRowIndex[id]; !ok {
return nil, roachpb.NewUErrorf("missing %q primary key column", tableDesc.PrimaryIndex.ColumnNames[i])
}
primaryKeyCols[id] = struct{}{}
}
// Construct the default expressions. The returned slice will be nil if no
// column in the table has a default expression.
defaultExprs, err := p.makeDefaultExprs(cols)
if err != nil {
return nil, roachpb.NewError(err)
}
// Replace any DEFAULT markers with the corresponding default expressions.
insertRows := p.fillDefaults(defaultExprs, cols, n)
// Transform the values into a rows object. This expands SELECT statements or
// generates rows from the values contained within the query.
rows, pErr := p.makePlan(insertRows, false)
if pErr != nil {
return nil, pErr
}
if expressions := len(rows.Columns()); expressions > numInputColumns {
return nil, roachpb.NewUErrorf("INSERT has more expressions than target columns: %d/%d", expressions, numInputColumns)
}
primaryIndex := tableDesc.PrimaryIndex
primaryIndexKeyPrefix := MakeIndexKeyPrefix(tableDesc.ID, primaryIndex.ID)
marshalled := make([]interface{}, len(cols))
b := p.txn.NewBatch()
rh, err := makeReturningHelper(p, n.Returning, tableDesc.Name, cols)
if err != nil {
return nil, roachpb.NewError(err)
}
for rows.Next() {
rowVals := rows.Values()
// The values for the row may be shorter than the number of columns being
// inserted into. Generate default values for those columns using the
// default expressions.
for i := len(rowVals); i < len(cols); i++ {
if defaultExprs == nil {
rowVals = append(rowVals, parser.DNull)
continue
}
d, err := defaultExprs[i].Eval(p.evalCtx)
if err != nil {
return nil, roachpb.NewError(err)
}
rowVals = append(rowVals, d)
}
// Check to see if NULL is being inserted into any non-nullable column.
for _, col := range tableDesc.Columns {
if !col.Nullable {
if i, ok := colIDtoRowIndex[col.ID]; !ok || rowVals[i] == parser.DNull {
return nil, roachpb.NewUErrorf("null value in column %q violates not-null constraint", col.Name)
}
}
}
// Check that the row value types match the column types. This needs to
// happen before index encoding because certain datum types (i.e. tuple)
// cannot be used as index values.
for i, val := range rowVals {
// Make sure the value can be written to the column before proceeding.
var mErr error
if marshalled[i], mErr = marshalColumnValue(cols[i], val, p.evalCtx.Args); mErr != nil {
return nil, roachpb.NewError(mErr)
}
}
if p.prepareOnly {
continue
}
primaryIndexKey, _, eErr := encodeIndexKey(
&primaryIndex, colIDtoRowIndex, rowVals, primaryIndexKeyPrefix)
if eErr != nil {
return nil, roachpb.NewError(eErr)
}
// Write the secondary indexes.
indexes := tableDesc.Indexes
// Also include the secondary indexes in mutation state WRITE_ONLY.
for _, m := range tableDesc.Mutations {
if m.State == DescriptorMutation_WRITE_ONLY {
if index := m.GetIndex(); index != nil {
indexes = append(indexes, *index)
}
}
}
secondaryIndexEntries, eErr := encodeSecondaryIndexes(
tableDesc.ID, indexes, colIDtoRowIndex, rowVals)
if eErr != nil {
return nil, roachpb.NewError(eErr)
}
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("CPut %s -> %v", secondaryIndexEntry.key,
secondaryIndexEntry.value)
}
b.CPut(secondaryIndexEntry.key, secondaryIndexEntry.value, nil)
}
// Write the row sentinel.
sentinelKey := keys.MakeNonColumnKey(primaryIndexKey)
if log.V(2) {
log.Infof("CPut %s -> NULL", roachpb.Key(sentinelKey))
}
// This is subtle: An interface{}(nil) deletes the value, so we pass in
// []byte{} as a non-nil value.
b.CPut(sentinelKey, []byte{}, nil)
// Write the row columns.
for i, val := range rowVals {
col := cols[i]
if _, ok := primaryKeyCols[col.ID]; ok {
// Skip primary key columns as their values are encoded in the row
// sentinel key which is guaranteed to exist for as long as the row
// exists.
continue
}
if marshalled[i] != nil {
// We only output non-NULL values. Non-existent column keys are
// considered NULL during scanning and the row sentinel ensures we know
// the row exists.
key := keys.MakeColumnKey(primaryIndexKey, uint32(col.ID))
if log.V(2) {
log.Infof("CPut %s -> %v", roachpb.Key(key), val)
}
b.CPut(key, marshalled[i], nil)
}
}
if err := rh.append(rowVals); err != nil {
return nil, roachpb.NewError(err)
}
}
if pErr := rows.PErr(); pErr != nil {
return nil, pErr
}
if p.prepareOnly {
// Return the result column types.
return rh.getResults(), nil
}
if isSystemConfigID(tableDesc.GetID()) {
// Mark transaction as operating on the system DB.
p.txn.SetSystemConfigTrigger()
}
if autoCommit {
// An auto-txn can commit the transaction with the batch. This is an
// optimization to avoid an extra round-trip to the transaction
// coordinator.
pErr = p.txn.CommitInBatch(b)
} else {
pErr = p.txn.Run(b)
}
if pErr != nil {
return nil, convertBatchError(&tableDesc, *b, pErr)
}
return rh.getResults(), nil
}
func (sc *SchemaChanger) truncateAndBackfillColumns(
lease *TableDescriptor_SchemaChangeLease,
added []ColumnDescriptor,
dropped []ColumnDescriptor,
version DescriptorVersion,
) *roachpb.Error {
evalCtx := parser.EvalContext{}
// Set the eval context timestamps.
pTime := timeutil.Now()
evalCtx.SetTxnTimestamp(pTime)
evalCtx.SetStmtTimestamp(pTime)
defaultExprs, err := makeDefaultExprs(added, &parser.Parser{}, evalCtx)
if err != nil {
return roachpb.NewError(err)
}
// Remember any new non nullable column with no default value.
nonNullableColumn := ""
for _, columnDesc := range added {
if columnDesc.DefaultExpr == nil && !columnDesc.Nullable {
nonNullableColumn = columnDesc.Name
}
}
// Add or Drop a column.
if len(dropped) > 0 || nonNullableColumn != "" || len(defaultExprs) > 0 {
// First extend the schema change lease.
l, pErr := sc.ExtendLease(*lease)
if pErr != nil {
return pErr
}
*lease = l
pErr = sc.db.Txn(func(txn *client.Txn) *roachpb.Error {
tableDesc, pErr := getTableDescAtVersion(txn, sc.tableID, version)
if pErr != nil {
return pErr
}
// Run a scan across the table using the primary key.
start := roachpb.Key(MakeIndexKeyPrefix(tableDesc.ID, tableDesc.PrimaryIndex.ID))
b := &client.Batch{}
b.Scan(start, start.PrefixEnd(), 0)
if pErr := txn.Run(b); pErr != nil {
return pErr
}
if nonNullableColumn != "" {
for _, result := range b.Results {
if len(result.Rows) > 0 {
return roachpb.NewErrorf("column %s contains null values", nonNullableColumn)
}
}
}
// Use a different batch to truncate/backfill columns.
writeBatch := &client.Batch{}
for _, result := range b.Results {
var sentinelKey roachpb.Key
for _, kv := range result.Rows {
if sentinelKey == nil || !bytes.HasPrefix(kv.Key, sentinelKey) {
// Sentinel keys have a 0 suffix indicating 0 bytes of column
// ID. Strip off that suffix to determine the prefix shared with the
// other keys for the row.
sentinelKey = stripColumnIDLength(kv.Key)
// Delete the entire dropped columns.
// This used to use SQL UPDATE in the past to update the dropped
// column to NULL; but a column in the process of being
// dropped is placed in the table descriptor mutations, and
// a SQL UPDATE of a column in mutations will fail.
for _, columnDesc := range dropped {
// Delete the dropped column.
colKey := keys.MakeColumnKey(sentinelKey, uint32(columnDesc.ID))
if log.V(2) {
log.Infof("Del %s", colKey)
}
writeBatch.Del(colKey)
}
// Add the new columns and backfill the values.
for i, expr := range defaultExprs {
if expr == nil {
continue
}
col := added[i]
colKey := keys.MakeColumnKey(sentinelKey, uint32(col.ID))
d, err := expr.Eval(evalCtx)
if err != nil {
return roachpb.NewError(err)
}
val, err := marshalColumnValue(col, d, evalCtx.Args)
if err != nil {
return roachpb.NewError(err)
}
if log.V(2) {
log.Infof("Put %s -> %v", colKey, val)
}
// Insert default value into the column. If this row
// was recently added the default value might have
// already been populated, because the
// ColumnDescriptor is in the WRITE_ONLY state.
// Reinserting the default value is not a big deal.
//
// Note: a column in the WRITE_ONLY state cannot be
// populated directly through SQL. A SQL INSERT cannot
// directly reference the column, and the INSERT
// populates the column with the default value.
writeBatch.Put(colKey, val)
}
}
}
}
if pErr := txn.Run(writeBatch); pErr != nil {
return convertBackfillError(tableDesc, writeBatch, pErr)
}
return nil
})
return pErr
}
return nil
}
// Update updates columns for a selection of rows from a table.
// Privileges: UPDATE and SELECT on table. We currently always use a select statement.
// Notes: postgres requires UPDATE. Requires SELECT with WHERE clause with table.
// mysql requires UPDATE. Also requires SELECT with WHERE clause with table.
func (p *planner) Update(n *parser.Update) (planNode, *roachpb.Error) {
tableDesc, pErr := p.getAliasedTableLease(n.Table)
if pErr != nil {
return nil, pErr
}
if err := p.checkPrivilege(tableDesc, privilege.UPDATE); err != nil {
return nil, roachpb.NewError(err)
}
// Determine which columns we're inserting into.
var names parser.QualifiedNames
for _, expr := range n.Exprs {
var epErr *roachpb.Error
expr.Expr, epErr = p.expandSubqueries(expr.Expr, len(expr.Names))
if epErr != nil {
return nil, epErr
}
if expr.Tuple {
// TODO(pmattis): The distinction between Tuple and DTuple here is
// irritating. We'll see a DTuple if the expression was a subquery that
// has been evaluated. We'll see a Tuple in other cases.
n := 0
switch t := expr.Expr.(type) {
case parser.Tuple:
n = len(t)
case parser.DTuple:
n = len(t)
default:
return nil, roachpb.NewErrorf("unsupported tuple assignment: %T", expr.Expr)
}
if len(expr.Names) != n {
return nil, roachpb.NewUErrorf("number of columns (%d) does not match number of values (%d)",
len(expr.Names), n)
}
}
names = append(names, expr.Names...)
}
cols, err := p.processColumns(tableDesc, names)
if err != nil {
return nil, roachpb.NewError(err)
}
// Set of columns being updated
colIDSet := map[ColumnID]struct{}{}
for _, c := range cols {
colIDSet[c.ID] = struct{}{}
}
// Don't allow updating any column that is part of the primary key.
for i, id := range tableDesc.PrimaryIndex.ColumnIDs {
if _, ok := colIDSet[id]; ok {
return nil, roachpb.NewUErrorf("primary key column %q cannot be updated", tableDesc.PrimaryIndex.ColumnNames[i])
}
}
defaultExprs, err := p.makeDefaultExprs(cols)
if err != nil {
return nil, roachpb.NewError(err)
}
// Generate the list of select targets. We need to select all of the columns
// plus we select all of the update expressions in case those expressions
// reference columns (e.g. "UPDATE t SET v = v + 1"). Note that we flatten
// expressions for tuple assignments just as we flattened the column names
// above. So "UPDATE t SET (a, b) = (1, 2)" translates into select targets of
// "*, 1, 2", not "*, (1, 2)".
targets := tableDesc.allColumnsSelector()
i := 0
for _, expr := range n.Exprs {
if expr.Tuple {
switch t := expr.Expr.(type) {
case parser.Tuple:
for _, e := range t {
e = fillDefault(e, i, defaultExprs)
targets = append(targets, parser.SelectExpr{Expr: e})
i++
}
case parser.DTuple:
for _, e := range t {
targets = append(targets, parser.SelectExpr{Expr: e})
i++
}
}
} else {
e := fillDefault(expr.Expr, i, defaultExprs)
targets = append(targets, parser.SelectExpr{Expr: e})
i++
}
}
// Query the rows that need updating.
rows, pErr := p.Select(&parser.Select{
Exprs: targets,
From: parser.TableExprs{n.Table},
Where: n.Where,
})
if pErr != nil {
return nil, pErr
}
// ValArgs have their types populated in the above Select if they are part
// of an expression ("SET a = 2 + $1") in the type check step where those
// types are inferred. For the simpler case ("SET a = $1"), populate them
// using marshalColumnValue. This step also verifies that the expression
// types match the column types.
if p.prepareOnly {
i := 0
f := func(expr parser.Expr) *roachpb.Error {
idx := i
i++
// DefaultVal doesn't implement TypeCheck
if _, ok := expr.(parser.DefaultVal); ok {
return nil
}
d, err := expr.TypeCheck(p.evalCtx.Args)
if err != nil {
return roachpb.NewError(err)
}
if _, err := marshalColumnValue(cols[idx], d, p.evalCtx.Args); err != nil {
return roachpb.NewError(err)
}
return nil
}
for _, expr := range n.Exprs {
if expr.Tuple {
switch t := expr.Expr.(type) {
case parser.Tuple:
for _, e := range t {
if err := f(e); err != nil {
return nil, err
}
}
case parser.DTuple:
for _, e := range t {
if err := f(e); err != nil {
return nil, err
}
}
}
} else {
if err := f(expr.Expr); err != nil {
return nil, err
}
}
}
return nil, nil
}
// Construct a map from column ID to the index the value appears at within a
// row.
colIDtoRowIndex := map[ColumnID]int{}
for i, col := range tableDesc.Columns {
colIDtoRowIndex[col.ID] = i
}
primaryIndex := tableDesc.PrimaryIndex
primaryIndexKeyPrefix := MakeIndexKeyPrefix(tableDesc.ID, primaryIndex.ID)
// Secondary indexes needing updating.
needsUpdate := func(index IndexDescriptor) bool {
for _, id := range index.ColumnIDs {
if _, ok := colIDSet[id]; ok {
return true
}
}
return false
}
indexes := make([]IndexDescriptor, 0, len(tableDesc.Indexes)+len(tableDesc.Mutations))
var deleteOnlyIndex map[int]struct{}
for _, index := range tableDesc.Indexes {
if needsUpdate(index) {
indexes = append(indexes, index)
}
}
for _, m := range tableDesc.Mutations {
if index := m.GetIndex(); index != nil {
if needsUpdate(*index) {
indexes = append(indexes, *index)
switch m.State {
case DescriptorMutation_DELETE_ONLY:
if deleteOnlyIndex == nil {
// Allocate at most once.
deleteOnlyIndex = make(map[int]struct{}, len(tableDesc.Mutations))
}
deleteOnlyIndex[len(indexes)-1] = struct{}{}
case DescriptorMutation_WRITE_ONLY:
}
}
}
}
marshalled := make([]interface{}, len(cols))
b := client.Batch{}
result := &valuesNode{}
for rows.Next() {
rowVals := rows.Values()
result.rows = append(result.rows, parser.DTuple(nil))
primaryIndexKey, _, err := encodeIndexKey(
&primaryIndex, colIDtoRowIndex, rowVals, primaryIndexKeyPrefix)
if err != nil {
return nil, roachpb.NewError(err)
}
// Compute the current secondary index key:value pairs for this row.
secondaryIndexEntries, err := encodeSecondaryIndexes(
tableDesc.ID, indexes, colIDtoRowIndex, rowVals)
if err != nil {
return nil, roachpb.NewError(err)
}
// Our updated value expressions occur immediately after the plain
// columns in the output.
newVals := rowVals[len(tableDesc.Columns):]
// Update the row values.
for i, col := range cols {
val := newVals[i]
if !col.Nullable && val == parser.DNull {
return nil, roachpb.NewUErrorf("null value in column %q violates not-null constraint", col.Name)
}
rowVals[colIDtoRowIndex[col.ID]] = val
}
// Check that the new value types match the column types. This needs to
// happen before index encoding because certain datum types (i.e. tuple)
// cannot be used as index values.
for i, val := range newVals {
var mErr error
if marshalled[i], mErr = marshalColumnValue(cols[i], val, p.evalCtx.Args); mErr != nil {
return nil, roachpb.NewError(mErr)
}
}
// Compute the new secondary index key:value pairs for this row.
newSecondaryIndexEntries, eErr := encodeSecondaryIndexes(
tableDesc.ID, indexes, colIDtoRowIndex, rowVals)
if eErr != nil {
return nil, roachpb.NewError(eErr)
}
// Update secondary indexes.
for i, newSecondaryIndexEntry := range newSecondaryIndexEntries {
secondaryIndexEntry := secondaryIndexEntries[i]
if !bytes.Equal(newSecondaryIndexEntry.key, secondaryIndexEntry.key) {
// Do not update Indexes in the DELETE_ONLY state.
if _, ok := deleteOnlyIndex[i]; !ok {
if log.V(2) {
log.Infof("CPut %s -> %v", newSecondaryIndexEntry.key,
newSecondaryIndexEntry.value)
}
b.CPut(newSecondaryIndexEntry.key, newSecondaryIndexEntry.value, nil)
}
if log.V(2) {
log.Infof("Del %s", secondaryIndexEntry.key)
}
b.Del(secondaryIndexEntry.key)
}
}
// Add the new values.
for i, val := range newVals {
col := cols[i]
key := keys.MakeColumnKey(primaryIndexKey, uint32(col.ID))
if marshalled[i] != nil {
// We only output non-NULL values. Non-existent column keys are
// considered NULL during scanning and the row sentinel ensures we know
// the row exists.
if log.V(2) {
log.Infof("Put %s -> %v", key, val)
}
b.Put(key, marshalled[i])
} else {
// The column might have already existed but is being set to NULL, so
// delete it.
if log.V(2) {
log.Infof("Del %s", key)
}
b.Del(key)
}
}
}
if pErr := rows.PErr(); pErr != nil {
return nil, pErr
}
if pErr := p.txn.Run(&b); pErr != nil {
return nil, convertBatchError(tableDesc, b, pErr)
}
return result, nil
}
// Update updates columns for a selection of rows from a table.
// Privileges: UPDATE and SELECT on table. We currently always use a select statement.
// Notes: postgres requires UPDATE. Requires SELECT with WHERE clause with table.
// mysql requires UPDATE. Also requires SELECT with WHERE clause with table.
// TODO(guanqun): need to support CHECK in UPDATE
func (p *planner) Update(n *parser.Update, autoCommit bool) (planNode, *roachpb.Error) {
tracing.AnnotateTrace()
tableDesc, pErr := p.getAliasedTableLease(n.Table)
if pErr != nil {
return nil, pErr
}
if err := p.checkPrivilege(tableDesc, privilege.UPDATE); err != nil {
return nil, roachpb.NewError(err)
}
// TODO(dan): Consider caching this on the TableDescriptor.
primaryKeyCols := map[ColumnID]struct{}{}
for _, id := range tableDesc.PrimaryIndex.ColumnIDs {
primaryKeyCols[id] = struct{}{}
}
exprs := make([]parser.UpdateExpr, len(n.Exprs))
for i, expr := range n.Exprs {
exprs[i] = *expr
}
// Determine which columns we're inserting into.
var names parser.QualifiedNames
for i, expr := range exprs {
newExpr, epErr := p.expandSubqueries(expr.Expr, len(expr.Names))
if epErr != nil {
return nil, epErr
}
exprs[i].Expr = newExpr
if expr.Tuple {
// TODO(pmattis): The distinction between Tuple and DTuple here is
// irritating. We'll see a DTuple if the expression was a subquery that
// has been evaluated. We'll see a Tuple in other cases.
n := 0
switch t := newExpr.(type) {
case *parser.Tuple:
n = len(t.Exprs)
case parser.DTuple:
n = len(t)
default:
return nil, roachpb.NewErrorf("unsupported tuple assignment: %T", newExpr)
}
if len(expr.Names) != n {
return nil, roachpb.NewUErrorf("number of columns (%d) does not match number of values (%d)",
len(expr.Names), n)
}
}
names = append(names, expr.Names...)
}
cols, err := p.processColumns(tableDesc, names)
if err != nil {
return nil, roachpb.NewError(err)
}
// Set of columns being updated
var primaryKeyColChange bool
colIDSet := map[ColumnID]struct{}{}
for _, c := range cols {
colIDSet[c.ID] = struct{}{}
if _, ok := primaryKeyCols[c.ID]; ok {
primaryKeyColChange = true
}
}
defaultExprs, err := makeDefaultExprs(cols, &p.parser, p.evalCtx)
if err != nil {
return nil, roachpb.NewError(err)
}
// Generate the list of select targets. We need to select all of the columns
// plus we select all of the update expressions in case those expressions
// reference columns (e.g. "UPDATE t SET v = v + 1"). Note that we flatten
// expressions for tuple assignments just as we flattened the column names
// above. So "UPDATE t SET (a, b) = (1, 2)" translates into select targets of
// "*, 1, 2", not "*, (1, 2)".
// TODO(radu): we only need to select columns necessary to generate primary and
// secondary indexes keys, and columns needed by returningHelper.
targets := tableDesc.allColumnsSelector()
i := 0
// Remember the index where the targets for exprs start.
exprTargetIdx := len(targets)
for _, expr := range exprs {
if expr.Tuple {
switch t := expr.Expr.(type) {
case *parser.Tuple:
for _, e := range t.Exprs {
e = fillDefault(e, i, defaultExprs)
targets = append(targets, parser.SelectExpr{Expr: e})
i++
}
case parser.DTuple:
for _, e := range t {
targets = append(targets, parser.SelectExpr{Expr: e})
i++
}
}
} else {
e := fillDefault(expr.Expr, i, defaultExprs)
targets = append(targets, parser.SelectExpr{Expr: e})
i++
}
}
tracing.AnnotateTrace()
// Query the rows that need updating.
rows, pErr := p.SelectClause(&parser.SelectClause{
Exprs: targets,
From: []parser.TableExpr{n.Table},
Where: n.Where,
})
if pErr != nil {
return nil, pErr
}
rh, err := makeReturningHelper(p, n.Returning, tableDesc.Name, tableDesc.Columns)
if err != nil {
return nil, roachpb.NewError(err)
}
// ValArgs have their types populated in the above Select if they are part
// of an expression ("SET a = 2 + $1") in the type check step where those
// types are inferred. For the simpler case ("SET a = $1"), populate them
// using marshalColumnValue. This step also verifies that the expression
// types match the column types.
if p.evalCtx.PrepareOnly {
for i, target := range rows.(*selectNode).render[exprTargetIdx:] {
// DefaultVal doesn't implement TypeCheck
if _, ok := target.(parser.DefaultVal); ok {
continue
}
d, err := target.TypeCheck(p.evalCtx.Args)
if err != nil {
return nil, roachpb.NewError(err)
}
if _, err := marshalColumnValue(cols[i], d, p.evalCtx.Args); err != nil {
return nil, roachpb.NewError(err)
}
}
// Return the result column types.
return rh.getResults()
}
// Construct a map from column ID to the index the value appears at within a
// row.
colIDtoRowIndex := map[ColumnID]int{}
for i, col := range tableDesc.Columns {
colIDtoRowIndex[col.ID] = i
}
primaryIndex := tableDesc.PrimaryIndex
primaryIndexKeyPrefix := MakeIndexKeyPrefix(tableDesc.ID, primaryIndex.ID)
// Secondary indexes needing updating.
needsUpdate := func(index IndexDescriptor) bool {
// If the primary key changed, we need to update all of them.
if primaryKeyColChange {
return true
}
for _, id := range index.ColumnIDs {
if _, ok := colIDSet[id]; ok {
return true
}
}
return false
}
indexes := make([]IndexDescriptor, 0, len(tableDesc.Indexes)+len(tableDesc.Mutations))
var deleteOnlyIndex map[int]struct{}
for _, index := range tableDesc.Indexes {
if needsUpdate(index) {
indexes = append(indexes, index)
}
}
for _, m := range tableDesc.Mutations {
if index := m.GetIndex(); index != nil {
if needsUpdate(*index) {
indexes = append(indexes, *index)
switch m.State {
case DescriptorMutation_DELETE_ONLY:
if deleteOnlyIndex == nil {
// Allocate at most once.
deleteOnlyIndex = make(map[int]struct{}, len(tableDesc.Mutations))
}
deleteOnlyIndex[len(indexes)-1] = struct{}{}
case DescriptorMutation_WRITE_ONLY:
}
}
}
}
marshalled := make([]interface{}, len(cols))
b := p.txn.NewBatch()
tracing.AnnotateTrace()
for rows.Next() {
tracing.AnnotateTrace()
rowVals := rows.Values()
primaryIndexKey, _, err := encodeIndexKey(
&primaryIndex, colIDtoRowIndex, rowVals, primaryIndexKeyPrefix)
if err != nil {
return nil, roachpb.NewError(err)
}
// Compute the current secondary index key:value pairs for this row.
secondaryIndexEntries, err := encodeSecondaryIndexes(
tableDesc.ID, indexes, colIDtoRowIndex, rowVals)
if err != nil {
return nil, roachpb.NewError(err)
}
// Our updated value expressions occur immediately after the plain
// columns in the output.
newVals := rowVals[len(tableDesc.Columns):]
// Ensure that the values honor the specified column widths.
for i := range newVals {
if err := checkValueWidth(cols[i], newVals[i]); err != nil {
return nil, roachpb.NewError(err)
}
}
// Update the row values.
for i, col := range cols {
val := newVals[i]
if !col.Nullable && val == parser.DNull {
return nil, roachpb.NewUErrorf("null value in column %q violates not-null constraint", col.Name)
}
rowVals[colIDtoRowIndex[col.ID]] = val
}
// Check that the new value types match the column types. This needs to
// happen before index encoding because certain datum types (i.e. tuple)
// cannot be used as index values.
for i, val := range newVals {
var mErr error
if marshalled[i], mErr = marshalColumnValue(cols[i], val, p.evalCtx.Args); mErr != nil {
return nil, roachpb.NewError(mErr)
}
}
// Compute the new primary index key for this row.
newPrimaryIndexKey := primaryIndexKey
var rowPrimaryKeyChanged bool
if primaryKeyColChange {
newPrimaryIndexKey, _, err = encodeIndexKey(
&primaryIndex, colIDtoRowIndex, rowVals, primaryIndexKeyPrefix)
if err != nil {
return nil, roachpb.NewError(err)
}
// Note that even if primaryIndexColChange is true, it's possible that
// primary key fields in this particular row didn't change.
rowPrimaryKeyChanged = !bytes.Equal(primaryIndexKey, newPrimaryIndexKey)
}
// Compute the new secondary index key:value pairs for this row.
newSecondaryIndexEntries, eErr := encodeSecondaryIndexes(
tableDesc.ID, indexes, colIDtoRowIndex, rowVals)
if eErr != nil {
return nil, roachpb.NewError(eErr)
}
if rowPrimaryKeyChanged {
// Delete all the data stored under the old primary key.
rowStartKey := roachpb.Key(primaryIndexKey)
rowEndKey := rowStartKey.PrefixEnd()
if log.V(2) {
log.Infof("DelRange %s - %s", rowStartKey, rowEndKey)
}
b.DelRange(rowStartKey, rowEndKey, false)
// Delete all the old secondary indexes.
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("Del %s", secondaryIndexEntry.key)
}
b.Del(secondaryIndexEntry.key)
}
// Write the new row sentinel. We want to write the sentinel first in case
// we are trying to insert a duplicate primary key: if we write the
// secondary indexes first, we may get an error that looks like a
// uniqueness violation on a non-unique index.
sentinelKey := keys.MakeNonColumnKey(newPrimaryIndexKey)
if log.V(2) {
log.Infof("CPut %s -> NULL", roachpb.Key(sentinelKey))
}
// This is subtle: An interface{}(nil) deletes the value, so we pass in
// []byte{} as a non-nil value.
b.CPut(sentinelKey, []byte{}, nil)
// Write any fields from the old row that were not modified by the UPDATE.
for i, col := range tableDesc.Columns {
if _, ok := colIDSet[col.ID]; ok {
continue
}
if _, ok := primaryKeyCols[col.ID]; ok {
continue
}
key := keys.MakeColumnKey(newPrimaryIndexKey, uint32(col.ID))
val := rowVals[i]
marshalledVal, mErr := marshalColumnValue(col, val, p.evalCtx.Args)
if mErr != nil {
return nil, roachpb.NewError(mErr)
}
if log.V(2) {
log.Infof("Put %s -> %v", roachpb.Key(key), val)
}
b.Put(key, marshalledVal)
}
// At this point, we've deleted the old row and associated index data and
// written the sentinel keys and column keys for non-updated columns. Fall
// through to below where the index keys and updated column keys will be
// written.
}
// Update secondary indexes.
for i, newSecondaryIndexEntry := range newSecondaryIndexEntries {
secondaryIndexEntry := secondaryIndexEntries[i]
secondaryKeyChanged := !bytes.Equal(newSecondaryIndexEntry.key, secondaryIndexEntry.key)
if secondaryKeyChanged {
if log.V(2) {
log.Infof("Del %s", secondaryIndexEntry.key)
}
b.Del(secondaryIndexEntry.key)
}
if rowPrimaryKeyChanged || secondaryKeyChanged {
// Do not update Indexes in the DELETE_ONLY state.
if _, ok := deleteOnlyIndex[i]; !ok {
if log.V(2) {
log.Infof("CPut %s -> %v", newSecondaryIndexEntry.key,
newSecondaryIndexEntry.value)
}
b.CPut(newSecondaryIndexEntry.key, newSecondaryIndexEntry.value, nil)
}
}
}
// Add the new values.
for i, val := range newVals {
col := cols[i]
if _, ok := primaryKeyCols[col.ID]; ok {
// Skip primary key columns as their values are encoded in the row
// sentinel key which is guaranteed to exist for as long as the row
// exists.
continue
}
key := keys.MakeColumnKey(newPrimaryIndexKey, uint32(col.ID))
if marshalled[i] != nil {
// We only output non-NULL values. Non-existent column keys are
// considered NULL during scanning and the row sentinel ensures we know
// the row exists.
if log.V(2) {
log.Infof("Put %s -> %v", roachpb.Key(key), val)
}
b.Put(key, marshalled[i])
} else {
// The column might have already existed but is being set to NULL, so
// delete it.
if log.V(2) {
log.Infof("Del %s", key)
}
b.Del(key)
}
}
// rowVals[:len(tableDesc.Columns)] have been updated with the new values above.
if err := rh.append(rowVals[:len(tableDesc.Columns)]); err != nil {
return nil, roachpb.NewError(err)
}
}
tracing.AnnotateTrace()
if pErr := rows.PErr(); pErr != nil {
return nil, pErr
}
if isSystemConfigID(tableDesc.GetID()) {
// Mark transaction as operating on the system DB.
p.txn.SetSystemConfigTrigger()
}
if autoCommit {
// An auto-txn can commit the transaction with the batch. This is an
// optimization to avoid an extra round-trip to the transaction
// coordinator.
pErr = p.txn.CommitInBatch(b)
} else {
pErr = p.txn.Run(b)
}
if pErr != nil {
return nil, convertBatchError(tableDesc, *b, pErr)
}
tracing.AnnotateTrace()
return rh.getResults()
}