// Test that we fail to lease a table that was marked for deletion. func TestCantLeaseDeletedTable(testingT *testing.T) { defer leaktest.AfterTest(testingT)() defer csql.TestDisableAsyncSchemaChangeExec()() var execKnobs csql.ExecutorTestingKnobs var mu sync.Mutex clearSchemaChangers := false execKnobs.SyncSchemaChangersFilter = func(tscc csql.TestingSchemaChangerCollection) { mu.Lock() defer mu.Unlock() if clearSchemaChangers { tscc.ClearSchemaChangers() } } ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLExecutor = &execKnobs t := newLeaseTest(testingT, ctx) defer t.cleanup() sql := ` CREATE DATABASE test; CREATE TABLE test.t(a INT PRIMARY KEY); ` _, err := t.db.Exec(sql) if err != nil { t.Fatal(err) } // Block schema changers so that the table we're about to DROP is not actually // dropped; it will be left in a "deleted" state. mu.Lock() clearSchemaChangers = true mu.Unlock() // DROP the table _, err = t.db.Exec(`DROP TABLE test.t`) if err != nil { t.Fatal(err) } // Make sure we can't get a lease on the descriptor. tableDesc := getTableDescriptor(t.kvDB, "test", "t") // try to acquire at a bogus version to make sure we don't get back a lease we // already had. _, err = t.acquire(1, tableDesc.ID, tableDesc.Version+1) if !testutils.IsError(err, "descriptor deleted") { t.Fatalf("got a different error than expected: %s", err) } }
// Test schema change backfills are not affected by various operations // that run simultaneously. func TestRaceWithBackfill(t *testing.T) { defer leaktest.AfterTest(t)() // Disable asynchronous schema change execution to allow synchronous path // to trigger start of backfill notification. defer csql.TestDisableAsyncSchemaChangeExec()() var execKnobs csql.ExecutorTestingKnobs var backfillNotification chan bool execKnobs.SchemaChangersStartBackfillNotification = func() { if backfillNotification != nil { // Close channel to notify that the backfill has started. close(backfillNotification) } } ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLExecutor = &execKnobs server, sqlDB, kvDB := setupWithContext(t, ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k INT PRIMARY KEY, v INT, pi DECIMAL DEFAULT (DECIMAL '3.14')); CREATE UNIQUE INDEX vidx ON t.test (v); `); err != nil { t.Fatal(err) } // Bulk insert. maxValue := 4000 insert := fmt.Sprintf(`INSERT INTO t.test VALUES (%d, %d)`, 0, maxValue) for i := 1; i <= maxValue; i++ { insert += fmt.Sprintf(` ,(%d, %d)`, i, maxValue-i) } if _, err := sqlDB.Exec(insert); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } tableDesc := desc.GetTable() tablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(tableDesc.ID))) tableEnd := tablePrefix.PrefixEnd() // number of keys == 4 * number of rows; 3 columns and 1 index entry for // each row. if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil { t.Fatal(err) } else if e := 4 * (maxValue + 1); len(kvs) != e { t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs)) } // Run some schema changes with operations. // Add column. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "ALTER TABLE t.test ADD COLUMN x DECIMAL DEFAULT (DECIMAL '1.4')", maxValue, 5, descKey, backfillNotification) // Drop column. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "ALTER TABLE t.test DROP pi", maxValue, 4, descKey, backfillNotification) // Add index. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "CREATE UNIQUE INDEX foo ON t.test (v)", maxValue, 5, descKey, backfillNotification) // Drop index. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "DROP INDEX t.test@vidx", maxValue, 4, descKey, backfillNotification) // Verify that the index foo over v is consistent, and that column x has // been backfilled properly. rows, err := sqlDB.Query(`SELECT v, x from t.test@foo`) if err != nil { t.Fatal(err) } count := 0 for ; rows.Next(); count++ { var val int var x float64 if err := rows.Scan(&val, &x); err != nil { t.Errorf("row %d scan failed: %s", count, err) continue } if count != val { t.Errorf("e = %d, v = %d", count, val) } if 1.4 != x { t.Errorf("e = %f, v = %f", 1.4, x) } } if err := rows.Err(); err != nil { t.Fatal(err) } eCount := maxValue + 1 if eCount != count { t.Fatalf("read the wrong number of rows: e = %d, v = %d", eCount, count) } // Verify that a table delete in the middle of a backfill works properly. // The backfill will terminate in the middle, and the delete will // successfully delete all the table data. // // This test could be made its own test but is placed here to speed up the // testing. backfillNotification = make(chan bool) // Run the schema change in a separate goroutine. var wg sync.WaitGroup wg.Add(1) go func() { // Start schema change that eventually runs a backfill. if _, err := sqlDB.Exec("CREATE UNIQUE INDEX bar ON t.test (v)"); err != nil { t.Error(err) } wg.Done() }() // Wait until the schema change backfill starts. <-backfillNotification // Wait for a short bit to ensure that the backfill has likely progressed // and written some data, but not long enough that the backfill has // completed. time.Sleep(10 * time.Millisecond) if _, err := sqlDB.Exec("DROP TABLE t.test"); err != nil { t.Fatal(err) } // Wait until the schema change is done. wg.Wait() // Ensure that the table data has been deleted. if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil { t.Fatal(err) } else if len(kvs) != 0 { t.Fatalf("expected %d key value pairs, but got %d", 0, len(kvs)) } }
func TestAsyncSchemaChanger(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable synchronous schema change execution so the asynchronous schema // changer executes all schema changes. var execKnobs csql.ExecutorTestingKnobs execKnobs.SyncSchemaChangersFilter = func(tscc csql.TestingSchemaChangerCollection) { tscc.ClearSchemaChangers() } defer csql.TestSpeedupAsyncSchemaChanges()() ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLExecutor = &execKnobs server, sqlDB, kvDB := setupWithContext(t, ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } // A long running schema change operation runs through // a state machine that increments the version by 3. expectedVersion := desc.GetTable().Version + 3 // Run some schema change if _, err := sqlDB.Exec(` CREATE INDEX foo ON t.test (v) `); err != nil { t.Fatal(err) } retryOpts := retry.Options{ InitialBackoff: 20 * time.Millisecond, MaxBackoff: 200 * time.Millisecond, Multiplier: 2, } // Wait until index is created. for r := retry.Start(retryOpts); r.Next(); { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } if len(desc.GetTable().Indexes) == 1 { break } } // Ensure that the indexes have been created. mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } indexQuery := `SELECT v FROM t.test@foo` _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) // Ensure that the version has been incremented. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion := desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Apply a schema change that only sets the UpVersion bit. expectedVersion = newVersion + 1 if _, err := sqlDB.Exec(` ALTER INDEX t.test@foo RENAME TO ufo `); err != nil { t.Fatal(err) } for r := retry.Start(retryOpts); r.Next(); { // Ensure that the version gets incremented. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } name := desc.GetTable().Indexes[0].Name if name != "ufo" { t.Fatalf("bad index name %s", name) } newVersion = desc.GetTable().Version if newVersion == expectedVersion { break } } // Run many schema changes simultaneously and check // that they all get executed. count := 5 for i := 0; i < count; i++ { cmd := fmt.Sprintf(`CREATE INDEX foo%d ON t.test (v)`, i) if _, err := sqlDB.Exec(cmd); err != nil { t.Fatal(err) } } // Wait until indexes are created. for r := retry.Start(retryOpts); r.Next(); { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } if len(desc.GetTable().Indexes) == count+1 { break } } for i := 0; i < count; i++ { indexQuery := fmt.Sprintf(`SELECT v FROM t.test@foo%d`, i) _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) } }
// Test that once a table is marked as deleted, a lease's refcount dropping to 0 // means the lease is released immediately, as opposed to being released only // when it expires. func TestLeasesOnDeletedTableAreReleasedImmediately(t *testing.T) { defer leaktest.AfterTest(t)() defer csql.TestDisableAsyncSchemaChangeExec()() var execKnobs csql.ExecutorTestingKnobs var lmKnobs csql.LeaseManagerTestingKnobs var mu sync.Mutex clearSchemaChangers := false execKnobs.SyncSchemaChangersFilter = func(tscc csql.TestingSchemaChangerCollection) { mu.Lock() defer mu.Unlock() if clearSchemaChangers { tscc.ClearSchemaChangers() } } var waitTableID sqlbase.ID deleted := make(chan bool) lmKnobs.TestingLeasesRefreshedEvent = func(cfg config.SystemConfig) { mu.Lock() defer mu.Unlock() if waitTableID != 0 { if isDeleted(waitTableID, cfg) { close(deleted) waitTableID = 0 } } } ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLExecutor = &execKnobs ctx.TestingKnobs.SQLLeaseManager = &lmKnobs s, db, kvDB := setupWithContext(t, ctx) defer cleanup(s, db) sql := ` CREATE DATABASE test; CREATE TABLE test.t(a INT PRIMARY KEY); ` _, err := db.Exec(sql) if err != nil { t.Fatal(err) } tableDesc := getTableDescriptor(kvDB, "test", "t") lease1, err := acquire(s.TestServer, tableDesc.ID, 0) if err != nil { t.Fatal(err) } lease2, err := acquire(s.TestServer, tableDesc.ID, 0) if err != nil { t.Fatal(err) } // Block schema changers so that the table we're about to DROP is not actually // dropped; it will be left in a "deleted" state. // Also install a way to wait for the config update to be processed. mu.Lock() clearSchemaChangers = true waitTableID = tableDesc.ID mu.Unlock() // DROP the table _, err = db.Exec(`DROP TABLE test.t`) if err != nil { t.Fatal(err) } // Block until the LeaseManager has processed the gossip update. <-deleted // We should still be able to acquire, because we have an active lease. lease3, err := acquire(s.TestServer, tableDesc.ID, 0) if err != nil { t.Fatal(err) } // Release everything. if err := s.LeaseManager().Release(lease1); err != nil { t.Fatal(err) } if err := s.LeaseManager().Release(lease2); err != nil { t.Fatal(err) } if err := s.LeaseManager().Release(lease3); err != nil { t.Fatal(err) } // Now we shouldn't be able to acquire any more. _, err = acquire(s.TestServer, tableDesc.ID, 0) if !testutils.IsError(err, "descriptor deleted") { t.Fatalf("got a different error than expected: %s", err) } }