// TestSchemaChangeCommandsWithPendingMutations tests how schema change // commands behave when they are referencing schema elements that are // mutations that are not yet live. func TestSchemaChangeCommandsWithPendingMutations(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. params, _ := createTestServerParams() params.Knobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := serverutils.StartServer(t, params) defer server.Stopper().Stop() if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (a CHAR PRIMARY KEY, b CHAR, c CHAR, INDEX foo (c)); `); err != nil { t.Fatal(err) } // Read table descriptor tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") mt := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, tableDesc: tableDesc, } // Test CREATE INDEX in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`CREATE INDEX foo ON t.test (c)`); !testutils.IsError(err, `index "foo" being dropped, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`CREATE INDEX foo ON t.test (c)`); !testutils.IsError(err, `duplicate index name: "foo"`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Add column DROP mutation "b" mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`CREATE INDEX bar ON t.test (b)`); !testutils.IsError(err, `index "bar" contains unknown column "b"`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) // An index referencing a column mutation that is being added // is allowed to be added. if _, err := sqlDB.Exec(`CREATE INDEX bar ON t.test (b)`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test DROP INDEX in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) // Noop. if _, err := sqlDB.Exec(`DROP INDEX t.test@foo`); err != nil { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`DROP INDEX t.test@foo`); !testutils.IsError(err, `index "foo" in the middle of being added, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Test ALTER TABLE ADD/DROP column in the presence of mutations. // Add column DROP mutation "b" mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD b CHAR`); !testutils.IsError(err, `column "b" being dropped, try again later`) { t.Fatal(err) } // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP b`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD b CHAR`); !testutils.IsError(err, `duplicate column name: "b"`) { t.Fatal(err) } if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP b`); !testutils.IsError(err, `column "b" in the middle of being added, try again later`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test ALTER TABLE ADD CONSTRAINT in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT foo UNIQUE (c)`); !testutils.IsError(err, `index "foo" being dropped, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT foo UNIQUE (c)`); !testutils.IsError(err, `duplicate index name: "foo"`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Add column mutation "b" mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT bar UNIQUE (b)`); !testutils.IsError(err, `index "bar" contains unknown column "b"`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT bar UNIQUE (b)`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test DROP CONSTRAINT in the presence of mutations. // Add index mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP CONSTRAINT foo`); err != nil { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP CONSTRAINT foo`); !testutils.IsError(err, `constraint "foo" in the middle of being added, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Rename column/index, while index is under mutation. // Add index mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{}) if _, err := sqlDB.Exec(`ALTER INDEX t.test@foo RENAME to ufo`); err != nil { mt.Fatal(err) } if _, err := sqlDB.Exec(`ALTER TABLE t.test RENAME COLUMN c TO d`); err != nil { mt.Fatal(err) } // The mutation in the table descriptor has changed and we would like // to update our copy to make it live. mt.tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") // Make "ufo" live. mt.makeMutationsActive() // The index has been renamed to ufo, and the column to d. _ = mt.checkQueryResponse("SHOW INDEXES FROM t.test", [][]string{{"test", "primary", "true", "1", "a", "ASC", "false"}, {"test", "ufo", "false", "1", "d", "ASC", "false"}}) // Rename column under mutation works properly. // Add column mutation "b". mt.writeColumnMutation("b", sqlbase.DescriptorMutation{}) if _, err := sqlDB.Exec(`ALTER TABLE t.test RENAME COLUMN b TO e`); err != nil { mt.Fatal(err) } // The mutation in the table descriptor has changed and we would like // to update our copy to make it live. mt.tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") // Make column "e" live. mt.makeMutationsActive() // Column b changed to d. _ = mt.checkQueryResponse("SHOW COLUMNS FROM t.test", [][]string{{"a", "STRING", "false", "NULL"}, {"d", "STRING", "true", "NULL"}, {"e", "STRING", "true", "NULL"}}) // Try to change column defaults while column is under mutation. mt.writeColumnMutation("e", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ALTER COLUMN e SET DEFAULT 'a'`); !testutils.IsError( err, `column "e" in the middle of being added`) { t.Fatal(err) } mt.makeMutationsActive() mt.writeColumnMutation("e", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ALTER COLUMN e SET DEFAULT 'a'`); !testutils.IsError( err, `column "e" in the middle of being dropped`) { t.Fatal(err) } mt.makeMutationsActive() }
func TestOperationsWithIndexMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. params, _ := createTestServerParams() params.Knobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := serverutils.StartServer(t, params) defer server.Stopper().Stop() if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo (v)); `); err != nil { t.Fatal(err) } // read table descriptor tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, tableDesc: tableDesc, } starQuery := `SELECT * FROM t.test` indexQuery := `SELECT v FROM t.test@foo` // See the effect of the operations depending on the state. for _, state := range []sqlbase.DescriptorMutation_State{sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY} { // Init table with some entries. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z"}, {"b", "y"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2)`, row[0], row[1]); err != nil { t.Fatal(err) } } _ = mTest.checkQueryResponse(starQuery, initRows) // Index foo is visible. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) // Index foo is invisible once it's a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) if _, err := sqlDB.Query(indexQuery); !testutils.IsError(err, `index "foo" not found`) { t.Fatal(err) } // Insert a new entry. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"b", "y"}, {"c", "x"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() if state == sqlbase.DescriptorMutation_DELETE_ONLY { // "x" didn't get added to the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) } else { // "x" got added to the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"x"}, {"y"}, {"z"}}) } // Make "foo" a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Update. if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'w' WHERE k = 'c'`); err != nil { t.Fatal(err) } // Update "v" to its current value "z" in row "a". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'z' WHERE k = 'a'`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"b", "y"}, {"c", "w"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() if state == sqlbase.DescriptorMutation_DELETE_ONLY { // updating "x" -> "w" is a noop on the index, // updating "z" -> "z" results in "z" being deleted from the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) } else { // updating "x" -> "w" results in the index updating from "x" -> "w", // updating "z" -> "z" is a noop on the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"w"}, {"y"}, {"z"}}) } // Make "foo" a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Delete row "b". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'b'`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"c", "w"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() // Deleting row "b" deletes "y" from the index. if state == sqlbase.DescriptorMutation_DELETE_ONLY { mTest.checkQueryResponse(indexQuery, [][]string{{"z"}}) } else { mTest.checkQueryResponse(indexQuery, [][]string{{"w"}, {"z"}}) } } // Check that a mutation can only be inserted with an explicit mutation state. tableDesc = mTest.tableDesc tableDesc.Mutations = []sqlbase.DescriptorMutation{{Descriptor_: &sqlbase.DescriptorMutation_Index{Index: &tableDesc.Indexes[len(tableDesc.Indexes)-1]}}} tableDesc.Indexes = tableDesc.Indexes[:len(tableDesc.Indexes)-1] if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, index foo, id 2") { t.Fatal(err) } }
// TestOperationsWithUniqueColumnMutation tests all the operations while an // index mutation refers to a column mutation. func TestOperationsWithUniqueColumnMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. params, _ := createTestServerParams() params.Knobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := serverutils.StartServer(t, params) defer server.Stopper().Stop() // Create a table with column i and an index on v and i. if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, i CHAR, INDEX foo (i, v)); `); err != nil { t.Fatal(err) } // read table descriptor tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, tableDesc: tableDesc, } starQuery := `SELECT * FROM t.test` indexQuery := `SELECT i FROM t.test@foo` // Run the tests for both states for a column and an index. for _, state := range []sqlbase.DescriptorMutation_State{ sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY, } { for _, idxState := range []sqlbase.DescriptorMutation_State{ sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY, } { // Ignore the impossible column in DELETE_ONLY state while index // is in the WRITE_ONLY state. if state == sqlbase.DescriptorMutation_DELETE_ONLY && idxState == sqlbase.DescriptorMutation_WRITE_ONLY { continue } // Init table to start state. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z", "q"}, {"b", "y", "r"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2, $3)`, row[0], row[1], row[2]); err != nil { t.Fatal(err) } } // Check that the table only contains the initRows. _ = mTest.checkQueryResponse(starQuery, initRows) // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: idxState}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Insert a row into the table. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Error(err) } // Make column "i" and index "foo" live. mTest.makeMutationsActive() // column "i" has no entry. _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z", "q"}, {"b", "y", "r"}, {"c", "x", "NULL"}}) if idxState == sqlbase.DescriptorMutation_DELETE_ONLY { // No index entry for row "c" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"q"}, {"r"}}) } else { // Index entry for row "c" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}, {"r"}}) } // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: idxState}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Updating column "i" for a row fails. if _, err := sqlDB.Exec(`UPDATE t.test SET (v, i) = ('u', 'u') WHERE k = 'a'`); !testutils.IsError(err, `column "i" does not exist`) { t.Error(err) } // Update a row without specifying mutation column "i". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'u' WHERE k = 'a'`); err != nil { t.Error(err) } // Make column "i" and index "foo" live. mTest.makeMutationsActive() // The update to column "v" is seen; there is no effect on column "i". _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "u", "q"}, {"b", "y", "r"}, {"c", "x", "NULL"}}) if idxState == sqlbase.DescriptorMutation_DELETE_ONLY { // Index entry for row "a" is deleted. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"r"}}) } else { // No change in index "foo" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}, {"r"}}) } // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: idxState}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Delete row "b". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'b'`); err != nil { t.Error(err) } // Make column "i" and index "foo" live. mTest.makeMutationsActive() // Row "b" is deleted. numVals is the number of non-NULL values seen, // or the number of KV values belonging to all the rows in the table // excluding row "b" since it's deleted. numVals := mTest.checkQueryResponse(starQuery, [][]string{{"a", "u", "q"}, {"c", "x", "NULL"}}) // idxVals is the number of index values seen. var idxVals int if idxState == sqlbase.DescriptorMutation_DELETE_ONLY { // Index entry for row "b" is deleted. idxVals = mTest.checkQueryResponse(indexQuery, [][]string{}) } else { // Index entry for row "b" is deleted. idxVals doesn't account for // the NULL value seen. idxVals = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}}) // Increment idxVals to account for the NULL value seen above. idxVals++ } // Check that there are no hidden KV values for row "b", and column // "i" for row "b" was deleted. Also check that the index values are // all accounted for. mTest.checkTableSize(numVals + idxVals) } } }
func TestCommandsWithPendingMutations(t *testing.T) { defer leaktest.AfterTest(t) // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. defer csql.TestDisableAsyncSchemaChangeExec()() server, sqlDB, kvDB := setup(t) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (a CHAR PRIMARY KEY, b CHAR, c CHAR, INDEX foo (c)); `); err != nil { t.Fatal(err) } // Read table descriptor nameKey := csql.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := csql.MakeDescMetadataKey(csql.ID(gr.ValueInt())) desc := &csql.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mt := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } // Test CREATE INDEX in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`CREATE INDEX foo ON t.test (c)`); !testutils.IsError(err, `index "foo" being dropped, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`CREATE INDEX foo ON t.test (c)`); !testutils.IsError(err, `duplicate index name: "foo"`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Add column DROP mutation "b" mt.writeColumnMutation("b", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`CREATE INDEX bar ON t.test (b)`); !testutils.IsError(err, `index "bar" contains unknown column "b"`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) // An index referencing a column mutation that is being added // is allowed to be added. if _, err := sqlDB.Exec(`CREATE INDEX bar ON t.test (b)`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test DROP INDEX in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) // Noop. if _, err := sqlDB.Exec(`DROP INDEX t.test@foo`); err != nil { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`DROP INDEX t.test@foo`); !testutils.IsError(err, `index "foo" in the middle of being added, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Test ALTER TABLE ADD/DROP column in the presence of mutations. // Add column DROP mutation "b" mt.writeColumnMutation("b", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD b CHAR`); !testutils.IsError(err, `column "b" being dropped, try again later`) { t.Fatal(err) } // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP b`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD b CHAR`); !testutils.IsError(err, `duplicate column name: "b"`) { t.Fatal(err) } if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP b`); !testutils.IsError(err, `column "b" in the middle of being added, try again later`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test ALTER TABLE ADD CONSTRAINT in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT foo UNIQUE (c)`); !testutils.IsError(err, `index "foo" being dropped, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT foo UNIQUE (c)`); !testutils.IsError(err, `duplicate index name: "foo"`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Add column mutation "b" mt.writeColumnMutation("b", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT bar UNIQUE (b)`); !testutils.IsError(err, `index "bar" contains unknown column "b"`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT bar UNIQUE (b)`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test DROP CONSTRAINT in the presence of mutations. // Add index mutation "foo"" mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_DROP}) // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP CONSTRAINT foo`); err != nil { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", csql.DescriptorMutation{Direction: csql.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP CONSTRAINT foo`); !testutils.IsError(err, `constraint "foo" in the middle of being added, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Rename column/index, while index is under mutation. // Add index mutation "foo"" mt.writeIndexMutation("foo", csql.DescriptorMutation{}) if _, err := sqlDB.Exec(`ALTER INDEX t.test@foo RENAME to ufo`); err != nil { mt.Fatal(err) } if _, err := sqlDB.Exec(`ALTER TABLE t.test RENAME COLUMN c TO d`); err != nil { mt.Fatal(err) } // The mutation in the table descriptor has changed and we would like // to update our copy to make it live. if err := mt.kvDB.GetProto(mt.descKey, mt.desc); err != nil { mt.Fatal(err) } // Make "ufo" live. mt.makeMutationsActive() // The index has been renamed to ufo, and the column to d. _ = mt.checkQueryResponse("SHOW INDEX FROM t.test", [][]string{{"test", "primary", "true", "1", "a", "ASC", "false"}, {"test", "ufo", "false", "1", "d", "ASC", "false"}}) // Rename column under mutation works properly. // Add column mutation "b". mt.writeColumnMutation("b", csql.DescriptorMutation{}) if _, err := sqlDB.Exec(`ALTER TABLE t.test RENAME COLUMN b TO e`); err != nil { mt.Fatal(err) } // The mutation in the table descriptor has changed and we would like // to update our copy to make it live. if err := mt.kvDB.GetProto(mt.descKey, mt.desc); err != nil { mt.Fatal(err) } // Make column "e" live. mt.makeMutationsActive() // Column b changed to d. _ = mt.checkQueryResponse("SHOW COLUMNS FROM t.test", [][]string{{"a", "STRING", "true", "NULL"}, {"d", "STRING", "true", "NULL"}, {"e", "STRING", "true", "NULL"}}) }
func TestOperationsWithColumnMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. params, _ := createTestServerParams() params.Knobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := serverutils.StartServer(t, params) defer server.Stopper().Stop() if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, i CHAR DEFAULT 'i'); `); err != nil { t.Fatal(err) } // read table descriptor tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, tableDesc: tableDesc, } starQuery := `SELECT * FROM t.test` // Run the tests for both states. for _, state := range []sqlbase.DescriptorMutation_State{sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY} { // Init table to start state. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z", "q"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2, $3)`, row[0], row[1], row[2]); err != nil { t.Fatal(err) } } // Check that the table only contains the initRows. _ = mTest.checkQueryResponse(starQuery, initRows) // Add column "i" as a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // A direct read of column "i" fails. if _, err := sqlDB.Query(`SELECT i FROM t.test`); err == nil { t.Fatalf("Read succeeded despite column being in %v state", sqlbase.DescriptorMutation{State: state}) } // The table only contains columns "k" and "v". _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}}) // The column backfill uses Put instead of CPut because it depends on // an INSERT of a column in the WRITE_ONLY state failing. These two // tests guarantee that. // Inserting a row into the table while specifying column "i" results in an error. if _, err := sqlDB.Exec(`INSERT INTO t.test (k, v, i) VALUES ('b', 'y', 'i')`); !testutils.IsError(err, `column "i" does not exist`) { t.Fatal(err) } // Repeating the same without specifying the columns results in a different error. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('b', 'y', 'i')`); !testutils.IsError(err, "INSERT error: table t.test has 2 columns but 3 values were supplied") { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Check that we can read all the rows and columns. _ = mTest.checkQueryResponse(starQuery, initRows) var afterInsert, afterUpdate, afterDelete [][]string if state == sqlbase.DescriptorMutation_DELETE_ONLY { // The default value of "i" for column "i" is not written. afterInsert = [][]string{{"a", "z", "q"}, {"c", "x", "NULL"}} // Update is a noop for column "i". afterUpdate = [][]string{{"a", "u", "q"}, {"c", "x", "NULL"}} // Delete also deletes column "i". afterDelete = [][]string{{"c", "x", "NULL"}} } else { // The default value of "i" for column "i" is written. afterInsert = [][]string{{"a", "z", "q"}, {"c", "x", "i"}} // Update is a noop for column "i". afterUpdate = [][]string{{"a", "u", "q"}, {"c", "x", "i"}} // Delete also deletes column "i". afterDelete = [][]string{{"c", "x", "i"}} } // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Insert a row into the table. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Notice that the default value of "i" is only written when the // descriptor is in the WRITE_ONLY state. _ = mTest.checkQueryResponse(starQuery, afterInsert) // The column backfill uses Put instead of CPut because it depends on // an UPDATE of a column in the WRITE_ONLY state failing. This test // guarantees that. // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Updating column "i" for a row fails. if _, err := sqlDB.Exec(`UPDATE t.test SET (v, i) = ('u', 'u') WHERE k = 'a'`); !testutils.IsError(err, `column "i" does not exist`) { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // The above failed update was a noop. _ = mTest.checkQueryResponse(starQuery, afterInsert) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Update a row without specifying mutation column "i". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'u' WHERE k = 'a'`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // The update to column "v" is seen; there is no effect on column "i". _ = mTest.checkQueryResponse(starQuery, afterUpdate) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Delete row "a". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'a'`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Row "a" is deleted. numVals is the number of non-NULL values seen, // or the number of KV values belonging to all the rows in the table // excluding row "a" since it's deleted. numVals := mTest.checkQueryResponse(starQuery, afterDelete) // Check that there are no hidden KV values for row "a", // and column "i" for row "a" was deleted. mTest.checkTableSize(numVals) } // Check that a mutation can only be inserted with an explicit mutation state, and direction. tableDesc = mTest.tableDesc tableDesc.Mutations = []sqlbase.DescriptorMutation{{}} if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, and no column/index descriptor") { t.Fatal(err) } tableDesc.Mutations = []sqlbase.DescriptorMutation{{Descriptor_: &sqlbase.DescriptorMutation_Column{Column: &tableDesc.Columns[len(tableDesc.Columns)-1]}}} tableDesc.Columns = tableDesc.Columns[:len(tableDesc.Columns)-1] if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, col i, id 3") { t.Fatal(err) } tableDesc.Mutations[0].State = sqlbase.DescriptorMutation_DELETE_ONLY if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state DELETE_ONLY, direction NONE, col i, id 3") { t.Fatal(err) } tableDesc.Mutations[0].State = sqlbase.DescriptorMutation_UNKNOWN tableDesc.Mutations[0].Direction = sqlbase.DescriptorMutation_DROP if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction DROP, col i, id 3") { t.Fatal(err) } }
func TestOperationsWithColumnMutation(t *testing.T) { defer leaktest.AfterTest(t) // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. defer csql.TestDisableAsyncSchemaChangeExec()() server, sqlDB, kvDB := setup(t) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, i CHAR DEFAULT 'i'); `); err != nil { t.Fatal(err) } // read table descriptor nameKey := csql.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := csql.MakeDescMetadataKey(csql.ID(gr.ValueInt())) desc := &csql.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } starQuery := `SELECT * FROM t.test` // Run the tests for both states. for _, state := range []csql.DescriptorMutation_State{csql.DescriptorMutation_DELETE_ONLY, csql.DescriptorMutation_WRITE_ONLY} { // Init table to start state. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z", "q"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2, $3)`, row[0], row[1], row[2]); err != nil { t.Fatal(err) } } // Check that the table only contains the initRows. _ = mTest.checkQueryResponse(starQuery, initRows) // Add column "i" as a mutation. mTest.writeColumnMutation("i", csql.DescriptorMutation{State: state}) // A direct read of column "i" fails. if _, err := sqlDB.Query(`SELECT i FROM t.test`); err == nil { t.Fatalf("Read succeeded despite column being in %v state", csql.DescriptorMutation{State: state}) } // The table only contains columns "k" and "v". _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}}) // Inserting a row into the table while specifying column "i" results in an error. if _, err := sqlDB.Exec(`INSERT INTO t.test (k, v, i) VALUES ('b', 'y', 'i')`); !testutils.IsError(err, `column "i" does not exist`) { t.Fatal(err) } // Repeating the same without specifying the columns results in a different error. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('b', 'y', 'i')`); !testutils.IsError(err, "INSERT has more expressions than target columns: 3/2") { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Check that we can read all the rows and columns. _ = mTest.checkQueryResponse(starQuery, initRows) var afterInsert, afterUpdate, afterDelete [][]string if state == csql.DescriptorMutation_DELETE_ONLY { // The default value of "i" for column "i" is not written. afterInsert = [][]string{{"a", "z", "q"}, {"c", "x", "NULL"}} // Update is a noop for column "i". afterUpdate = [][]string{{"a", "u", "q"}, {"c", "x", "NULL"}} // Delete also deletes column "i". afterDelete = [][]string{{"c", "x", "NULL"}} } else { // The default value of "i" for column "i" is written. afterInsert = [][]string{{"a", "z", "q"}, {"c", "x", "i"}} // Update is a noop for column "i". afterUpdate = [][]string{{"a", "u", "q"}, {"c", "x", "i"}} // Delete also deletes column "i". afterDelete = [][]string{{"c", "x", "i"}} } // Make column "i" a mutation. mTest.writeColumnMutation("i", csql.DescriptorMutation{State: state}) // Insert a row into the table. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Notice that the default value of "i" is only written when the // descriptor is in the WRITE_ONLY state. _ = mTest.checkQueryResponse(starQuery, afterInsert) // Make column "i" a mutation. mTest.writeColumnMutation("i", csql.DescriptorMutation{State: state}) // Updating column "i" for a row fails. if _, err := sqlDB.Exec(`UPDATE t.test SET (v, i) = ('u', 'u') WHERE k = 'a'`); !testutils.IsError(err, `column "i" does not exist`) { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // The above failed update was a noop. _ = mTest.checkQueryResponse(starQuery, afterInsert) // Make column "i" a mutation. mTest.writeColumnMutation("i", csql.DescriptorMutation{State: state}) // Update a row without specifying mutation column "i". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'u' WHERE k = 'a'`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // The update to column "v" is seen; there is no effect on column "i". _ = mTest.checkQueryResponse(starQuery, afterUpdate) // Make column "i" a mutation. mTest.writeColumnMutation("i", csql.DescriptorMutation{State: state}) // Delete row "a". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'a'`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Row "a" is deleted. numVals is the number of non-NULL values seen, // or the number of KV values belonging to all the rows in the table // excluding row "a" since it's deleted. numVals := mTest.checkQueryResponse(starQuery, afterDelete) // Check that there are no hidden KV values for row "a", // and column "i" for row "a" was deleted. mTest.checkTableSize(numVals) } // Check that a mutation can only be inserted with an explicit mutation state, and direction. tableDesc := desc.GetTable() tableDesc.Mutations = []csql.DescriptorMutation{{}} if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, and no column/index descriptor") { t.Fatal(err) } tableDesc.Mutations = []csql.DescriptorMutation{{Descriptor_: &csql.DescriptorMutation_Column{Column: &tableDesc.Columns[len(tableDesc.Columns)-1]}}} tableDesc.Columns = tableDesc.Columns[:len(tableDesc.Columns)-1] if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, col i, id 3") { t.Fatal(err) } tableDesc.Mutations[0].State = csql.DescriptorMutation_DELETE_ONLY if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state DELETE_ONLY, direction NONE, col i, id 3") { t.Fatal(err) } tableDesc.Mutations[0].State = csql.DescriptorMutation_UNKNOWN tableDesc.Mutations[0].Direction = csql.DescriptorMutation_DROP if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction DROP, col i, id 3") { t.Fatal(err) } }
func TestOperationsWithIndexMutation(t *testing.T) { defer leaktest.AfterTest(t) // The descriptor changes made must have an immediate effect. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. defer csql.TestDisableAsyncSchemaChangeExec()() server, sqlDB, kvDB := setup(t) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo (v)); `); err != nil { t.Fatal(err) } // read table descriptor nameKey := csql.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := csql.MakeDescMetadataKey(csql.ID(gr.ValueInt())) desc := &csql.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } starQuery := `SELECT * FROM t.test` indexQuery := `SELECT * FROM t.test@foo` // See the effect of the operations depending on the state. for _, state := range []csql.DescriptorMutation_State{csql.DescriptorMutation_DELETE_ONLY, csql.DescriptorMutation_WRITE_ONLY} { // Init table with some entries. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z"}, {"b", "y"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2)`, row[0], row[1]); err != nil { t.Fatal(err) } } _ = mTest.checkQueryResponse(starQuery, initRows) // Index foo is visible. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) // Index foo is invisible once it's a mutation. mTest.writeIndexMutation("foo", csql.DescriptorMutation{State: state}) if _, err := sqlDB.Query(indexQuery); !testutils.IsError(err, `index "foo" not found`) { t.Fatal(err) } // Insert a new entry. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"b", "y"}, {"c", "x"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() if state == csql.DescriptorMutation_DELETE_ONLY { // "x" didn't get added to the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) } else { // "x" got added to the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"x"}, {"y"}, {"z"}}) } // Make "foo" a mutation. mTest.writeIndexMutation("foo", csql.DescriptorMutation{State: state}) // Update. if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'w' WHERE k = 'c'`); err != nil { t.Fatal(err) } // Update "v" to its current value "z" in row "a". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'z' WHERE k = 'a'`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"b", "y"}, {"c", "w"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() if state == csql.DescriptorMutation_DELETE_ONLY { // updating "x" -> "w" is a noop on the index, // updating "z" -> "z" results in "z" being deleted from the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) } else { // updating "x" -> "w" results in the index updating from "x" -> "w", // updating "z" -> "z" is a noop on the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"w"}, {"y"}, {"z"}}) } // Make "foo" a mutation. mTest.writeIndexMutation("foo", csql.DescriptorMutation{State: state}) // Delete row "b". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'b'`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"c", "w"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() // Deleting row "b" deletes "y" from the index. if state == csql.DescriptorMutation_DELETE_ONLY { mTest.checkQueryResponse(indexQuery, [][]string{{"z"}}) } else { mTest.checkQueryResponse(indexQuery, [][]string{{"w"}, {"z"}}) } } // Check that a mutation can only be inserted with an explicit mutation state. tableDesc := desc.GetTable() tableDesc.Mutations = []csql.DescriptorMutation{{Descriptor_: &csql.DescriptorMutation_Index{Index: &tableDesc.Indexes[len(tableDesc.Indexes)-1]}}} tableDesc.Indexes = tableDesc.Indexes[:len(tableDesc.Indexes)-1] if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, index foo, id 2") { t.Fatal(err) } }
// TestOperationsWithUniqueColumnMutation tests all the operations while an // index mutation refers to a column mutation. func TestOperationsWithUniqueColumnMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) // Create a table with column i and an index on v and i. if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, i CHAR, INDEX foo (i, v)); `); err != nil { t.Fatal(err) } // read table descriptor nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } starQuery := `SELECT * FROM t.test` indexQuery := `SELECT i FROM t.test@foo` // Run the tests for both states. for _, state := range []sqlbase.DescriptorMutation_State{ sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY, } { // Init table to start state. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z", "q"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2, $3)`, row[0], row[1], row[2]); err != nil { t.Fatal(err) } } // Check that the table only contains the initRows. _ = mTest.checkQueryResponse(starQuery, initRows) // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Insert a row into the table. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Error(err) } // Make column "i" and index "foo" live. mTest.makeMutationsActive() // column "i" has no entry. _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z", "q"}, {"c", "x", "NULL"}}) if state == sqlbase.DescriptorMutation_DELETE_ONLY { // No index entry for row "c" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"q"}}) } else { // Index entry for row "c" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}}) } // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Updating column "i" for a row fails. if _, err := sqlDB.Exec(`UPDATE t.test SET (v, i) = ('u', 'u') WHERE k = 'a'`); !testutils.IsError(err, `column "i" does not exist`) { t.Error(err) } // TODO(vivek): Fix #6691. // Update a row without specifying mutation column "i". //if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'u' WHERE k = 'a'`); err != nil { // t.Error(err) //} // Make column "i" and index "foo" live. mTest.makeMutationsActive() // The update to column "v" is seen; there is no effect on column "i". //_ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "u", "q"}, {"c", "x", "NULL"}}) // No change in index "foo" //if state == sqlbase.DescriptorMutation_DELETE_ONLY { // _ = mTest.checkQueryResponse(indexQuery, [][]string{{"q"}}) //} else { // _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}}) //} // Add index "foo" as a mutation. //mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Make column "i" a mutation. //mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Delete row "a". //if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'a'`); err != nil { // t.Error(err) //} // Make column "i" and index "foo" live. // mTest.makeMutationsActive() // Row "a" is deleted. numVals is the number of non-NULL values seen, // or the number of KV values belonging to all the rows in the table // excluding row "a" since it's deleted. //numVals := mTest.checkQueryResponse(starQuery, [][]string{{"c", "x", "NULL"}}) //if state == sqlbase.DescriptorMutation_DELETE_ONLY { // _ = mTest.checkQueryResponse(indexQuery, [][]string{}) //} else { // _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}}) //} // Check that there are no hidden KV values for row "a", // and column "i" for row "a" was deleted. //mTest.checkTableSize(numVals) } }
func TestAsyncSchemaChanger(t *testing.T) { defer leaktest.AfterTest(t) // Disable synchronous schema change execution so // the asynchronous schema changer executes all // schema changes. defer csql.TestDisableSyncSchemaChangeExec()() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() server, sqlDB, kvDB := setup(t) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := csql.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := csql.MakeDescMetadataKey(csql.ID(gr.ValueInt())) desc := &csql.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } // A long running schema change operation runs through // a state machine that increments the version by 3. expectedVersion := desc.GetTable().Version + 3 // Run some schema change if _, err := sqlDB.Exec(` CREATE INDEX foo ON t.test (v) `); err != nil { t.Fatal(err) } retryOpts := retry.Options{ InitialBackoff: 20 * time.Millisecond, MaxBackoff: 200 * time.Millisecond, Multiplier: 2, } // Wait until index is created. for r := retry.Start(retryOpts); r.Next(); { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } if len(desc.GetTable().Indexes) == 1 { break } } // Ensure that the indexes have been created. mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } indexQuery := `SELECT * FROM t.test@foo` _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) // Ensure that the version has been incremented. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion := desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Apply a schema change that only sets the UpVersion bit. expectedVersion = newVersion + 1 if _, err := sqlDB.Exec(` ALTER INDEX t.test@foo RENAME TO ufo `); err != nil { t.Fatal(err) } for r := retry.Start(retryOpts); r.Next(); { // Ensure that the version gets incremented. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } name := desc.GetTable().Indexes[0].Name if name != "ufo" { t.Fatalf("bad index name %s", name) } newVersion = desc.GetTable().Version if newVersion == expectedVersion { break } } // Run many schema changes simultaneously and check // that they all get executed. count := 5 for i := 0; i < count; i++ { cmd := fmt.Sprintf(`CREATE INDEX foo%d ON t.test (v)`, i) if _, err := sqlDB.Exec(cmd); err != nil { t.Fatal(err) } } // Wait until indexes are created. for r := retry.Start(retryOpts); r.Next(); { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } if len(desc.GetTable().Indexes) == count+1 { break } } for i := 0; i < count; i++ { indexQuery := fmt.Sprintf(`SELECT * FROM t.test@foo%d`, i) _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) } }
func TestSchemaChangeProcess(t *testing.T) { defer leaktest.AfterTest(t) // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. defer csql.TestDisableAsyncSchemaChangeExec()() server, sqlDB, kvDB := setup(t) defer cleanup(server, sqlDB) var id = csql.ID(keys.MaxReservedDescID + 2) var node = roachpb.NodeID(2) db := server.DB() leaseMgr := csql.NewLeaseManager(0, *db, hlc.NewClock(hlc.UnixNano)) changer := csql.NewSchemaChangerForTesting(id, 0, node, *db, leaseMgr) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v)); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := csql.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := csql.MakeDescMetadataKey(csql.ID(gr.ValueInt())) desc := &csql.Descriptor{} // Check that MaybeIncrementVersion doesn't increment the version // when the up_version bit is not set. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } expectedVersion := desc.GetTable().Version if err := changer.MaybeIncrementVersion(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion := desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } isDone, err := changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", desc.GetTable()) } // Check that MaybeIncrementVersion increments the version // correctly. expectedVersion++ desc.GetTable().UpVersion = true if err := kvDB.Put(descKey, desc); err != nil { t.Fatal(err) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } if err := changer.MaybeIncrementVersion(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion = desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", desc.GetTable()) } // Check that RunStateMachineBeforeBackfill doesn't do anything // if there are no mutations queued. if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion = desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Check that RunStateMachineBeforeBackfill functions properly. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } table := desc.GetTable() expectedVersion = table.Version // Make a copy of the index for use in a mutation. index := proto.Clone(&table.Indexes[0]).(*csql.IndexDescriptor) index.Name = "bar" index.ID = table.NextIndexID table.NextIndexID++ changer = csql.NewSchemaChangerForTesting(id, table.NextMutationID, node, *db, leaseMgr) table.Mutations = append(table.Mutations, csql.DescriptorMutation{ Descriptor_: &csql.DescriptorMutation_Index{Index: index}, Direction: csql.DescriptorMutation_ADD, State: csql.DescriptorMutation_DELETE_ONLY, MutationID: table.NextMutationID, }) table.NextMutationID++ // Run state machine in both directions. for _, direction := range []csql.DescriptorMutation_Direction{csql.DescriptorMutation_ADD, csql.DescriptorMutation_DROP} { table.Mutations[0].Direction = direction expectedVersion++ if err := kvDB.Put(descKey, desc); err != nil { t.Fatal(err) } // The expected end state. expectedState := csql.DescriptorMutation_WRITE_ONLY if direction == csql.DescriptorMutation_DROP { expectedState = csql.DescriptorMutation_DELETE_ONLY } // Run two times to ensure idempotency of operations. for i := 0; i < 2; i++ { if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } table = desc.GetTable() newVersion = table.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } state := table.Mutations[0].State if state != expectedState { t.Fatalf("bad state; e = %d, v = %d", expectedState, state) } } } // RunStateMachineBeforeBackfill() doesn't complete the schema change. isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } }
func TestAsyncSchemaChanger(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable synchronous schema change execution so the asynchronous schema // changer executes all schema changes. params, _ := createTestServerParams() params.Knobs = base.TestingKnobs{ SQLExecutor: &csql.ExecutorTestingKnobs{ SyncSchemaChangersFilter: func(tscc csql.TestingSchemaChangerCollection) { tscc.ClearSchemaChangers() }, }, SQLSchemaChangeManager: &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecQuickly: true, }, } s, sqlDB, kvDB := serverutils.StartServer(t, params) defer s.Stopper().Stop() if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") // A long running schema change operation runs through // a state machine that increments the version by 3. expectedVersion := tableDesc.Version + 3 // Run some schema change if _, err := sqlDB.Exec(` CREATE INDEX foo ON t.test (v) `); err != nil { t.Fatal(err) } retryOpts := retry.Options{ InitialBackoff: 20 * time.Millisecond, MaxBackoff: 200 * time.Millisecond, Multiplier: 2, } // Wait until index is created. for r := retry.Start(retryOpts); r.Next(); { tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") if len(tableDesc.Indexes) == 1 { break } } // Ensure that the indexes have been created. mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, tableDesc: tableDesc, } indexQuery := `SELECT v FROM t.test@foo` _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) // Ensure that the version has been incremented. tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion := tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Apply a schema change that only sets the UpVersion bit. expectedVersion = newVersion + 1 if _, err := sqlDB.Exec(` ALTER INDEX t.test@foo RENAME TO ufo `); err != nil { t.Fatal(err) } for r := retry.Start(retryOpts); r.Next(); { // Ensure that the version gets incremented. tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") name := tableDesc.Indexes[0].Name if name != "ufo" { t.Fatalf("bad index name %s", name) } newVersion = tableDesc.Version if newVersion == expectedVersion { break } } // Run many schema changes simultaneously and check // that they all get executed. count := 5 for i := 0; i < count; i++ { cmd := fmt.Sprintf(`CREATE INDEX foo%d ON t.test (v)`, i) if _, err := sqlDB.Exec(cmd); err != nil { t.Fatal(err) } } // Wait until indexes are created. for r := retry.Start(retryOpts); r.Next(); { tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") if len(tableDesc.Indexes) == count+1 { break } } for i := 0; i < count; i++ { indexQuery := fmt.Sprintf(`SELECT v FROM t.test@foo%d`, i) _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) } }
func TestSchemaChangeProcess(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() params, _ := createTestServerParams() // Disable external processing of mutations. params.Knobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } s, sqlDB, kvDB := serverutils.StartServer(t, params) defer s.Stopper().Stop() var id = sqlbase.ID(keys.MaxReservedDescID + 2) var node = roachpb.NodeID(2) stopper := stop.NewStopper() leaseMgr := csql.NewLeaseManager(0, *kvDB, hlc.NewClock(hlc.UnixNano), csql.LeaseManagerTestingKnobs{}, stopper) defer stopper.Stop() changer := csql.NewSchemaChangerForTesting(id, 0, node, *kvDB, leaseMgr) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v)); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") expectedVersion := tableDesc.Version desc, err := changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } tableDesc = desc.GetTable() newVersion := tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } isDone, err := changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc) } // Check that MaybeIncrementVersion increments the version // correctly. expectedVersion++ tableDesc.UpVersion = true if err := kvDB.Put( sqlbase.MakeDescMetadataKey(tableDesc.ID), sqlbase.WrapDescriptor(tableDesc), ); err != nil { t.Fatal(err) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } desc, err = changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } tableDesc = desc.GetTable() savedTableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion = tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version in returned desc; e = %d, v = %d", expectedVersion, newVersion) } newVersion = savedTableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version in saved desc; e = %d, v = %d", expectedVersion, newVersion) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", tableDesc) } // Check that RunStateMachineBeforeBackfill doesn't do anything // if there are no mutations queued. if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion = tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Check that RunStateMachineBeforeBackfill functions properly. expectedVersion = tableDesc.Version // Make a copy of the index for use in a mutation. index := protoutil.Clone(&tableDesc.Indexes[0]).(*sqlbase.IndexDescriptor) index.Name = "bar" index.ID = tableDesc.NextIndexID tableDesc.NextIndexID++ changer = csql.NewSchemaChangerForTesting(id, tableDesc.NextMutationID, node, *kvDB, leaseMgr) tableDesc.Mutations = append(tableDesc.Mutations, sqlbase.DescriptorMutation{ Descriptor_: &sqlbase.DescriptorMutation_Index{Index: index}, Direction: sqlbase.DescriptorMutation_ADD, State: sqlbase.DescriptorMutation_DELETE_ONLY, MutationID: tableDesc.NextMutationID, }) tableDesc.NextMutationID++ // Run state machine in both directions. for _, direction := range []sqlbase.DescriptorMutation_Direction{sqlbase.DescriptorMutation_ADD, sqlbase.DescriptorMutation_DROP} { tableDesc.Mutations[0].Direction = direction expectedVersion++ if err := kvDB.Put( sqlbase.MakeDescMetadataKey(tableDesc.ID), sqlbase.WrapDescriptor(tableDesc), ); err != nil { t.Fatal(err) } // The expected end state. expectedState := sqlbase.DescriptorMutation_WRITE_ONLY if direction == sqlbase.DescriptorMutation_DROP { expectedState = sqlbase.DescriptorMutation_DELETE_ONLY } // Run two times to ensure idempotency of operations. for i := 0; i < 2; i++ { if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } tableDesc = sqlbase.GetTableDescriptor(kvDB, "t", "test") newVersion = tableDesc.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } state := tableDesc.Mutations[0].State if state != expectedState { t.Fatalf("bad state; e = %d, v = %d", expectedState, state) } } } // RunStateMachineBeforeBackfill() doesn't complete the schema change. isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", tableDesc) } }