// Run a particular schema change and run some OLTP operations in parallel, as
// soon as the schema change starts executing its backfill.
func runSchemaChangeWithOperations(
t *testing.T,
sqlDB *gosql.DB,
kvDB *client.DB,
schemaChange string,
maxValue int,
keyMultiple int,
backfillNotification chan bool,
) {
tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
// Run the schema change in a separate goroutine.
var wg sync.WaitGroup
wg.Add(1)
go func() {
start := timeutil.Now()
// Start schema change that eventually runs a backfill.
if _, err := sqlDB.Exec(schemaChange); err != nil {
t.Error(err)
}
t.Logf("schema change %s took %v", schemaChange, timeutil.Since(start))
wg.Done()
}()
// Wait until the schema change backfill starts.
<-backfillNotification
// Run a variety of operations during the backfill.
// Grabbing a schema change lease on the table will fail, disallowing
// another schema change from being simultaneously executed.
sc := csql.NewSchemaChangerForTesting(tableDesc.ID, 0, 0, *kvDB, nil)
if l, err := sc.AcquireLease(); err == nil {
t.Fatalf("schema change lease acquisition on table %d succeeded: %v", tableDesc.ID, l)
}
// Update some rows.
var updatedKeys []int
for i := 0; i < 10; i++ {
k := rand.Intn(maxValue)
v := maxValue + i + 1
if _, err := sqlDB.Exec(`UPDATE t.test SET v = $1 WHERE k = $2`, v, k); err != nil {
t.Error(err)
}
updatedKeys = append(updatedKeys, k)
}
// Reupdate updated values back to what they were before.
for _, k := range updatedKeys {
if _, err := sqlDB.Exec(`UPDATE t.test SET v = $1 WHERE k = $2`, maxValue-k, k); err != nil {
t.Error(err)
}
}
// Delete some rows.
deleteStartKey := rand.Intn(maxValue - 10)
for i := 0; i < 10; i++ {
if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = $1`, deleteStartKey+i); err != nil {
t.Error(err)
}
}
// Reinsert deleted rows.
for i := 0; i < 10; i++ {
k := deleteStartKey + i
if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES($1, $2)`, k, maxValue-k); err != nil {
t.Error(err)
}
}
// Insert some new rows.
numInserts := 10
for i := 0; i < numInserts; i++ {
k := maxValue + i + 1
if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES($1, $1)`, k); err != nil {
t.Error(err)
}
}
wg.Wait() // for schema change to complete.
// Verify the number of keys left behind in the table to validate schema
// change operations.
tablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(tableDesc.ID)))
tableEnd := tablePrefix.PrefixEnd()
if kvs, err := kvDB.Scan(context.TODO(), tablePrefix, tableEnd, 0); err != nil {
t.Fatal(err)
} else if e := keyMultiple * (maxValue + numInserts + 1); len(kvs) != e {
for _, kv := range kvs {
t.Errorf("key %s, value %s", kv.Key, kv.Value)
}
t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs))
}
// Delete the rows inserted.
for i := 0; i < numInserts; i++ {
if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = $1`, maxValue+i+1); err != nil {
t.Error(err)
}
}
}
func TestSchemaChangeLease(t *testing.T) {
defer leaktest.AfterTest(t)()
params, _ := createTestServerParams()
s, sqlDB, kvDB := serverutils.StartServer(t, params)
defer s.Stopper().Stop()
// Set MinSchemaChangeLeaseDuration to always expire the lease.
minLeaseDuration := csql.MinSchemaChangeLeaseDuration
csql.MinSchemaChangeLeaseDuration = 2 * csql.SchemaChangeLeaseDuration
defer func() {
csql.MinSchemaChangeLeaseDuration = minLeaseDuration
}()
if _, err := sqlDB.Exec(`
CREATE DATABASE t;
CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR);
`); err != nil {
t.Fatal(err)
}
var lease sqlbase.TableDescriptor_SchemaChangeLease
var id = sqlbase.ID(keys.MaxReservedDescID + 2)
var node = roachpb.NodeID(2)
changer := csql.NewSchemaChangerForTesting(id, 0, node, *kvDB, nil)
// Acquire a lease.
lease, err := changer.AcquireLease()
if err != nil {
t.Fatal(err)
}
if !validExpirationTime(lease.ExpirationTime) {
t.Fatalf("invalid expiration time: %s", time.Unix(0, lease.ExpirationTime))
}
// Acquiring another lease will fail.
if _, err := changer.AcquireLease(); !testutils.IsError(
err, "an outstanding schema change lease exists",
) {
t.Fatal(err)
}
// Extend the lease.
newLease, err := changer.ExtendLease(lease)
if err != nil {
t.Fatal(err)
}
if !validExpirationTime(newLease.ExpirationTime) {
t.Fatalf("invalid expiration time: %s", time.Unix(0, newLease.ExpirationTime))
}
// The new lease is a brand new lease.
if newLease == lease {
t.Fatalf("lease was not extended: %v", lease)
}
// Extending an old lease fails.
if _, err := changer.ExtendLease(lease); !testutils.IsError(err, "table: .* has lease") {
t.Fatal(err)
}
// Releasing an old lease fails.
err = changer.ReleaseLease(lease)
if err == nil {
t.Fatal("releasing a old lease succeeded")
}
// Release lease.
err = changer.ReleaseLease(newLease)
if err != nil {
t.Fatal(err)
}
// Extending the lease fails.
_, err = changer.ExtendLease(newLease)
if err == nil {
t.Fatalf("was able to extend an already released lease: %d, %v", id, lease)
}
// acquiring the lease succeeds
lease, err = changer.AcquireLease()
if err != nil {
t.Fatal(err)
}
// Set MinSchemaChangeLeaseDuration to not expire the lease.
csql.MinSchemaChangeLeaseDuration = minLeaseDuration
newLease, err = changer.ExtendLease(lease)
if err != nil {
t.Fatal(err)
}
// The old lease is renewed.
if newLease != lease {
t.Fatalf("acquired new lease: %v, old lease: %v", newLease, lease)
}
}
func TestSchemaChangeProcess(t *testing.T) {
defer leaktest.AfterTest(t)()
// The descriptor changes made must have an immediate effect
// so disable leases on tables.
defer csql.TestDisableTableLeases()()
params, _ := createTestServerParams()
// Disable external processing of mutations.
params.Knobs.SQLSchemaChanger = &csql.SchemaChangerTestingKnobs{
AsyncExecNotification: asyncSchemaChangerDisabled,
}
s, sqlDB, kvDB := serverutils.StartServer(t, params)
defer s.Stopper().Stop()
var id = sqlbase.ID(keys.MaxReservedDescID + 2)
var node = roachpb.NodeID(2)
stopper := stop.NewStopper()
leaseMgr := csql.NewLeaseManager(
&base.NodeIDContainer{},
*kvDB,
hlc.NewClock(hlc.UnixNano, time.Nanosecond),
csql.LeaseManagerTestingKnobs{},
stopper,
&csql.MemoryMetrics{},
)
defer stopper.Stop()
changer := csql.NewSchemaChangerForTesting(id, 0, node, *kvDB, leaseMgr)
if _, err := sqlDB.Exec(`
CREATE DATABASE t;
CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v));
INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd');
`); err != nil {
t.Fatal(err)
}
// Read table descriptor for version.
tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
expectedVersion := tableDesc.Version
desc, err := changer.MaybeIncrementVersion()
if err != nil {
t.Fatal(err)
}
tableDesc = desc.GetTable()
newVersion := tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion)
}
isDone, err := changer.IsDone()
if err != nil {
t.Fatal(err)
}
if !isDone {
t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc)
}
// Check that MaybeIncrementVersion increments the version
// correctly.
expectedVersion++
tableDesc.UpVersion = true
if err := kvDB.Put(
context.TODO(),
sqlbase.MakeDescMetadataKey(tableDesc.ID),
sqlbase.WrapDescriptor(tableDesc),
); err != nil {
t.Fatal(err)
}
isDone, err = changer.IsDone()
if err != nil {
t.Fatal(err)
}
if isDone {
t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable())
}
desc, err = changer.MaybeIncrementVersion()
if err != nil {
t.Fatal(err)
}
tableDesc = desc.GetTable()
savedTableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
newVersion = tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version in returned desc; e = %d, v = %d", expectedVersion, newVersion)
}
newVersion = savedTableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version in saved desc; e = %d, v = %d", expectedVersion, newVersion)
}
isDone, err = changer.IsDone()
if err != nil {
t.Fatal(err)
}
if !isDone {
t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc)
}
// Check that RunStateMachineBeforeBackfill doesn't do anything
// if there are no mutations queued.
if err := changer.RunStateMachineBeforeBackfill(); err != nil {
t.Fatal(err)
}
tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test")
newVersion = tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion)
}
// Check that RunStateMachineBeforeBackfill functions properly.
expectedVersion = tableDesc.Version
// Make a copy of the index for use in a mutation.
index := protoutil.Clone(&tableDesc.Indexes[0]).(*sqlbase.IndexDescriptor)
index.Name = "bar"
index.ID = tableDesc.NextIndexID
tableDesc.NextIndexID++
changer = csql.NewSchemaChangerForTesting(id, tableDesc.NextMutationID, node, *kvDB, leaseMgr)
tableDesc.Mutations = append(tableDesc.Mutations, sqlbase.DescriptorMutation{
Descriptor_: &sqlbase.DescriptorMutation_Index{Index: index},
Direction: sqlbase.DescriptorMutation_ADD,
State: sqlbase.DescriptorMutation_DELETE_ONLY,
MutationID: tableDesc.NextMutationID,
})
tableDesc.NextMutationID++
// Run state machine in both directions.
for _, direction := range []sqlbase.DescriptorMutation_Direction{sqlbase.DescriptorMutation_ADD, sqlbase.DescriptorMutation_DROP} {
tableDesc.Mutations[0].Direction = direction
expectedVersion++
if err := kvDB.Put(
context.TODO(),
sqlbase.MakeDescMetadataKey(tableDesc.ID),
sqlbase.WrapDescriptor(tableDesc),
); err != nil {
t.Fatal(err)
}
// The expected end state.
expectedState := sqlbase.DescriptorMutation_WRITE_ONLY
if direction == sqlbase.DescriptorMutation_DROP {
expectedState = sqlbase.DescriptorMutation_DELETE_ONLY
}
// Run two times to ensure idempotency of operations.
for i := 0; i < 2; i++ {
if err := changer.RunStateMachineBeforeBackfill(); err != nil {
t.Fatal(err)
}
tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test")
newVersion = tableDesc.Version
if newVersion != expectedVersion {
t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion)
}
state := tableDesc.Mutations[0].State
if state != expectedState {
t.Fatalf("bad state; e = %d, v = %d", expectedState, state)
}
}
}
// RunStateMachineBeforeBackfill() doesn't complete the schema change.
isDone, err = changer.IsDone()
if err != nil {
t.Fatal(err)
}
if isDone {
t.Fatalf("table expected to have an outstanding schema change: %v", tableDesc)
}
}