// encodeSecondaryIndexes encodes the secondary index keys. The
// secondaryIndexEntries are only valid until the next call to encodeIndexes or
// encodeSecondaryIndexes.
func (rh *rowHelper) encodeSecondaryIndexes(
colIDtoRowIndex map[sqlbase.ColumnID]int, values []parser.Datum,
) (
secondaryIndexEntries []sqlbase.IndexEntry,
err error,
) {
if len(rh.indexEntries) != len(rh.indexes) {
rh.indexEntries = make([]sqlbase.IndexEntry, len(rh.indexes))
}
err = sqlbase.EncodeSecondaryIndexes(
rh.tableDesc, rh.indexes, colIDtoRowIndex, values, rh.indexEntries)
if err != nil {
return nil, err
}
return rh.indexEntries, nil
}
func (rh *rowHelper) encodeIndexes(
colIDtoRowIndex map[sqlbase.ColumnID]int, values []parser.Datum,
) (
primaryIndexKey []byte,
secondaryIndexEntries []sqlbase.IndexEntry,
err error,
) {
if rh.primaryIndexKeyPrefix == nil {
rh.primaryIndexKeyPrefix = sqlbase.MakeIndexKeyPrefix(rh.tableDesc.ID,
rh.tableDesc.PrimaryIndex.ID)
}
primaryIndexKey, _, err = sqlbase.EncodeIndexKey(
&rh.tableDesc.PrimaryIndex, colIDtoRowIndex, values, rh.primaryIndexKeyPrefix)
if err != nil {
return nil, nil, err
}
secondaryIndexEntries, err = sqlbase.EncodeSecondaryIndexes(
rh.tableDesc.ID, rh.indexes, colIDtoRowIndex, values)
if err != nil {
return nil, nil, err
}
return primaryIndexKey, secondaryIndexEntries, nil
}
// updateRow adds to the batch the kv operations necessary to update a table row
// with the given values.
//
// The row corresponding to oldValues is updated with the ones in updateValues.
// Note that updateValues only contains the ones that are changing.
//
// The return value is only good until the next call to UpdateRow.
func (ru *rowUpdater) updateRow(
b *client.Batch,
oldValues []parser.Datum,
updateValues []parser.Datum,
) ([]parser.Datum, error) {
if len(oldValues) != len(ru.fetchCols) {
return nil, util.Errorf("got %d values but expected %d", len(oldValues), len(ru.fetchCols))
}
if len(updateValues) != len(ru.updateCols) {
return nil, util.Errorf("got %d values but expected %d", len(updateValues), len(ru.updateCols))
}
primaryIndexKey, secondaryIndexEntries, err := ru.helper.encodeIndexes(ru.fetchColIDtoRowIndex, oldValues)
if err != nil {
return nil, err
}
// Check that the new value types match the column types. This needs to
// happen before index encoding because certain datum types (i.e. tuple)
// cannot be used as index values.
for i, val := range updateValues {
if ru.marshalled[i], err = sqlbase.MarshalColumnValue(ru.updateCols[i], val); err != nil {
return nil, err
}
}
// Update the row values.
copy(ru.newValues, oldValues)
for i, updateCol := range ru.updateCols {
ru.newValues[ru.fetchColIDtoRowIndex[updateCol.ID]] = updateValues[i]
}
newPrimaryIndexKey := primaryIndexKey
rowPrimaryKeyChanged := false
var newSecondaryIndexEntries []sqlbase.IndexEntry
if ru.primaryKeyColChange {
newPrimaryIndexKey, newSecondaryIndexEntries, err = ru.helper.encodeIndexes(ru.fetchColIDtoRowIndex, ru.newValues)
if err != nil {
return nil, err
}
rowPrimaryKeyChanged = !bytes.Equal(primaryIndexKey, newPrimaryIndexKey)
} else {
newSecondaryIndexEntries, err = sqlbase.EncodeSecondaryIndexes(
ru.helper.tableDesc.ID, ru.helper.indexes, ru.fetchColIDtoRowIndex, ru.newValues)
if err != nil {
return nil, err
}
}
if rowPrimaryKeyChanged {
err := ru.rd.deleteRow(b, oldValues)
if err != nil {
return nil, err
}
err = ru.ri.insertRow(b, ru.newValues)
return ru.newValues, err
}
// Update secondary indexes.
for i, newSecondaryIndexEntry := range newSecondaryIndexEntries {
secondaryIndexEntry := secondaryIndexEntries[i]
secondaryKeyChanged := !bytes.Equal(newSecondaryIndexEntry.Key, secondaryIndexEntry.Key)
if secondaryKeyChanged {
if log.V(2) {
log.Infof("Del %s", secondaryIndexEntry.Key)
}
b.Del(secondaryIndexEntry.Key)
// Do not update Indexes in the DELETE_ONLY state.
if _, ok := ru.deleteOnlyIndex[i]; !ok {
if log.V(2) {
log.Infof("CPut %s -> %v", newSecondaryIndexEntry.Key, newSecondaryIndexEntry.Value)
}
b.CPut(newSecondaryIndexEntry.Key, newSecondaryIndexEntry.Value, nil)
}
}
}
// Add the new values.
for i, val := range updateValues {
col := ru.updateCols[i]
if ru.helper.columnInPK(col.ID) {
// Skip primary key columns as their values are encoded in the row
// sentinel key which is guaranteed to exist for as long as the row
// exists.
continue
}
ru.key = keys.MakeColumnKey(newPrimaryIndexKey, uint32(col.ID))
if ru.marshalled[i].RawBytes != nil {
// We only output non-NULL values. Non-existent column keys are
// considered NULL during scanning and the row sentinel ensures we know
// the row exists.
if log.V(2) {
log.Infof("Put %s -> %v", ru.key, val)
}
b.Put(&ru.key, &ru.marshalled[i])
} else {
// The column might have already existed but is being set to NULL, so
// delete it.
if log.V(2) {
log.Infof("Del %s", ru.key)
}
b.Del(&ru.key)
}
ru.key = nil
}
return ru.newValues, nil
}
func (sc *SchemaChanger) backfillIndexesChunk(
added []sqlbase.IndexDescriptor,
sp sqlbase.Span,
) (roachpb.Key, bool, error) {
var nextKey roachpb.Key
done := false
err := sc.db.Txn(func(txn *client.Txn) error {
tableDesc, err := getTableDescFromID(txn, sc.tableID)
if err != nil {
return err
}
// Short circuit the backfill if the table has been deleted.
if tableDesc.Deleted {
done = true
return nil
}
// Get the next set of rows.
// TODO(tamird): Support partial indexes?
//
// Use a scanNode with SELECT to pass in a sqlbase.TableDescriptor to the
// SELECT without needing to use a parser.QualifiedName, because we
// want to run schema changes from a gossip feed of table IDs. Running
// the scan and applying the changes in many transactions is fine
// because the schema change is in the correct state to handle
// intermediate OLTP commands which delete and add values during the
// scan.
scan := &scanNode{
planner: makePlanner(),
txn: txn,
desc: *tableDesc,
spans: []sqlbase.Span{sp},
}
scan.initDescDefaults()
rows, err := selectIndex(scan, nil, false)
if err != nil {
return err
}
// Construct a map from column ID to the index the value appears at
// within a row.
colIDtoRowIndex, err := makeColIDtoRowIndex(rows, tableDesc)
if err != nil {
return err
}
b := &client.Batch{}
numRows := 0
for ; numRows < IndexBackfillChunkSize && rows.Next(); numRows++ {
rowVals := rows.Values()
for _, desc := range added {
secondaryIndexEntries, err := sqlbase.EncodeSecondaryIndexes(
tableDesc.ID, []sqlbase.IndexDescriptor{desc}, colIDtoRowIndex, rowVals)
if err != nil {
return err
}
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("InitPut %s -> %v", secondaryIndexEntry.Key,
secondaryIndexEntry.Value)
}
b.InitPut(secondaryIndexEntry.Key, secondaryIndexEntry.Value)
}
}
}
if rows.Err() != nil {
return rows.Err()
}
// Write the new index values.
if err := txn.Run(b); err != nil {
for _, r := range b.Results {
if r.PErr != nil {
return convertBackfillError(tableDesc, b, r.PErr)
}
}
return err
}
// Have we processed all the table rows?
if numRows < IndexBackfillChunkSize {
done = true
return nil
}
// Keep track of the next key.
nextKey = scan.fetcher.Key()
return nil
})
return nextKey, done, err
}
func (sc *SchemaChanger) backfillIndexesChunk(
added []sqlbase.IndexDescriptor,
sp sqlbase.Span,
) (roachpb.Key, bool, error) {
var nextKey roachpb.Key
done := false
err := sc.db.Txn(context.TODO(), func(txn *client.Txn) error {
tableDesc, err := sqlbase.GetTableDescFromID(txn, sc.tableID)
if err != nil {
return err
}
// Short circuit the backfill if the table has been deleted.
if tableDesc.Deleted() {
done = true
return nil
}
// Get the next set of rows.
// TODO(tamird): Support partial indexes?
//
// Use a scanNode with SELECT to pass in a sqlbase.TableDescriptor
// to the SELECT without needing to go through table name
// resolution, because we want to run schema changes from a gossip
// feed of table IDs. Running the scan and applying the changes in
// many transactions is fine because the schema change is in the
// correct state to handle intermediate OLTP commands which delete
// and add values during the scan.
planner := makePlanner()
planner.setTxn(txn)
scan := planner.Scan()
scan.desc = *tableDesc
scan.spans = []sqlbase.Span{sp}
scan.initDescDefaults(publicAndNonPublicColumns)
rows, err := selectIndex(scan, nil, false)
if err != nil {
return err
}
if err := rows.Start(); err != nil {
return err
}
// Construct a map from column ID to the index the value appears at
// within a row.
colIDtoRowIndex, err := makeColIDtoRowIndex(rows, tableDesc)
if err != nil {
return err
}
b := &client.Batch{}
numRows := 0
for ; numRows < IndexBackfillChunkSize; numRows++ {
if next, err := rows.Next(); !next {
if err != nil {
return err
}
break
}
rowVals := rows.Values()
for _, desc := range added {
secondaryIndexEntries := make([]sqlbase.IndexEntry, 1)
err := sqlbase.EncodeSecondaryIndexes(
tableDesc, []sqlbase.IndexDescriptor{desc}, colIDtoRowIndex,
rowVals, secondaryIndexEntries)
if err != nil {
return err
}
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof(context.TODO(), "InitPut %s -> %v", secondaryIndexEntry.Key,
secondaryIndexEntry.Value)
}
b.InitPut(secondaryIndexEntry.Key, &secondaryIndexEntry.Value)
}
}
}
// Write the new index values.
if err := txn.Run(b); err != nil {
return convertBackfillError(tableDesc, b)
}
// Have we processed all the table rows?
if numRows < IndexBackfillChunkSize {
done = true
return nil
}
// Keep track of the next key.
nextKey = scan.fetcher.Key()
return nil
})
return nextKey, done, err
}