// Test schema change backfills are not affected by various operations
// that run simultaneously.
func TestRaceWithBackfill(t *testing.T) {
defer leaktest.AfterTest(t)()
// Disable asynchronous schema change execution to allow synchronous path
// to trigger start of backfill notification.
defer csql.TestDisableAsyncSchemaChangeExec()()
var execKnobs csql.ExecutorTestingKnobs
var backfillNotification chan bool
execKnobs.SchemaChangersStartBackfillNotification =
func() {
if backfillNotification != nil {
// Close channel to notify that the backfill has started.
close(backfillNotification)
}
}
ctx, _ := createTestServerContext()
ctx.TestingKnobs.SQLExecutor = &execKnobs
server, sqlDB, kvDB := setupWithContext(t, ctx)
defer cleanup(server, sqlDB)
if _, err := sqlDB.Exec(`
CREATE DATABASE t;
CREATE TABLE t.test (k INT PRIMARY KEY, v INT, pi DECIMAL DEFAULT (DECIMAL '3.14'));
CREATE UNIQUE INDEX vidx ON t.test (v);
`); err != nil {
t.Fatal(err)
}
// Bulk insert.
maxValue := 4000
insert := fmt.Sprintf(`INSERT INTO t.test VALUES (%d, %d)`, 0, maxValue)
for i := 1; i <= maxValue; i++ {
insert += fmt.Sprintf(` ,(%d, %d)`, i, maxValue-i)
}
if _, err := sqlDB.Exec(insert); err != nil {
t.Fatal(err)
}
// Read table descriptor for version.
nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test")
gr, err := kvDB.Get(nameKey)
if err != nil {
t.Fatal(err)
}
if !gr.Exists() {
t.Fatalf("Name entry %q does not exist", nameKey)
}
descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt()))
desc := &sqlbase.Descriptor{}
if err := kvDB.GetProto(descKey, desc); err != nil {
t.Fatal(err)
}
tableDesc := desc.GetTable()
tablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(tableDesc.ID)))
tableEnd := tablePrefix.PrefixEnd()
// number of keys == 4 * number of rows; 3 columns and 1 index entry for
// each row.
if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil {
t.Fatal(err)
} else if e := 4 * (maxValue + 1); len(kvs) != e {
t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs))
}
// Run some schema changes with operations.
// Add column.
backfillNotification = make(chan bool)
runSchemaChangeWithOperations(
t,
sqlDB,
kvDB,
"ALTER TABLE t.test ADD COLUMN x DECIMAL DEFAULT (DECIMAL '1.4')",
maxValue,
5,
descKey,
backfillNotification)
// Drop column.
backfillNotification = make(chan bool)
runSchemaChangeWithOperations(
t,
sqlDB,
kvDB,
"ALTER TABLE t.test DROP pi",
maxValue,
4,
descKey,
backfillNotification)
// Add index.
backfillNotification = make(chan bool)
runSchemaChangeWithOperations(
t,
sqlDB,
kvDB,
"CREATE UNIQUE INDEX foo ON t.test (v)",
maxValue,
5,
descKey,
backfillNotification)
// Drop index.
backfillNotification = make(chan bool)
runSchemaChangeWithOperations(
t,
sqlDB,
kvDB,
"DROP INDEX t.test@vidx",
maxValue,
4,
descKey,
backfillNotification)
// Verify that the index foo over v is consistent, and that column x has
// been backfilled properly.
rows, err := sqlDB.Query(`SELECT v, x from t.test@foo`)
if err != nil {
t.Fatal(err)
}
count := 0
for ; rows.Next(); count++ {
var val int
var x float64
if err := rows.Scan(&val, &x); err != nil {
t.Errorf("row %d scan failed: %s", count, err)
continue
}
if count != val {
t.Errorf("e = %d, v = %d", count, val)
}
if 1.4 != x {
t.Errorf("e = %f, v = %f", 1.4, x)
}
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
eCount := maxValue + 1
if eCount != count {
t.Fatalf("read the wrong number of rows: e = %d, v = %d", eCount, count)
}
// Verify that a table delete in the middle of a backfill works properly.
// The backfill will terminate in the middle, and the delete will
// successfully delete all the table data.
//
// This test could be made its own test but is placed here to speed up the
// testing.
backfillNotification = make(chan bool)
// Run the schema change in a separate goroutine.
var wg sync.WaitGroup
wg.Add(1)
go func() {
// Start schema change that eventually runs a backfill.
if _, err := sqlDB.Exec("CREATE UNIQUE INDEX bar ON t.test (v)"); err != nil {
t.Error(err)
}
wg.Done()
}()
// Wait until the schema change backfill starts.
<-backfillNotification
// Wait for a short bit to ensure that the backfill has likely progressed
// and written some data, but not long enough that the backfill has
// completed.
time.Sleep(10 * time.Millisecond)
if _, err := sqlDB.Exec("DROP TABLE t.test"); err != nil {
t.Fatal(err)
}
// Wait until the schema change is done.
wg.Wait()
// Ensure that the table data has been deleted.
if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil {
t.Fatal(err)
} else if len(kvs) != 0 {
t.Fatalf("expected %d key value pairs, but got %d", 0, len(kvs))
}
}
