func getTableDescriptor(db *client.DB, database string, table string) *sqlbase.TableDescriptor { dbNameKey := sqlbase.MakeNameMetadataKey(keys.RootNamespaceID, database) gr, err := db.Get(dbNameKey) if err != nil { panic(err) } if !gr.Exists() { panic("database missing") } dbDescID := sqlbase.ID(gr.ValueInt()) tableNameKey := sqlbase.MakeNameMetadataKey(dbDescID, table) gr, err = db.Get(tableNameKey) if err != nil { panic(err) } if !gr.Exists() { panic("table missing") } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := db.GetProto(descKey, desc); err != nil { panic("proto missing") } return desc.GetTable() }
// getTableNames implements the SchemaAccessor interface. func (p *planner) getTableNames(dbDesc *sqlbase.DatabaseDescriptor) (parser.QualifiedNames, error) { prefix := sqlbase.MakeNameMetadataKey(dbDesc.ID, "") sr, err := p.txn.Scan(prefix, prefix.PrefixEnd(), 0) if err != nil { return nil, err } var qualifiedNames parser.QualifiedNames for _, row := range sr { _, tableName, err := encoding.DecodeUnsafeStringAscending( bytes.TrimPrefix(row.Key, prefix), nil) if err != nil { return nil, err } qname := &parser.QualifiedName{ Base: parser.Name(dbDesc.Name), Indirect: parser.Indirection{parser.NameIndirection(tableName)}, } if err := qname.NormalizeTableName(""); err != nil { return nil, err } qualifiedNames = append(qualifiedNames, qname) } return qualifiedNames, nil }
// getTableNames implements the SchemaAccessor interface. func (p *planner) getTableNames(dbDesc *sqlbase.DatabaseDescriptor) (parser.TableNames, error) { if e, ok := getVirtualSchemaEntry(dbDesc.Name); ok { return e.tableNames(), nil } prefix := sqlbase.MakeNameMetadataKey(dbDesc.ID, "") sr, err := p.txn.Scan(prefix, prefix.PrefixEnd(), 0) if err != nil { return nil, err } var tableNames parser.TableNames for _, row := range sr { _, tableName, err := encoding.DecodeUnsafeStringAscending( bytes.TrimPrefix(row.Key, prefix), nil) if err != nil { return nil, err } tn := parser.TableName{ DatabaseName: parser.Name(dbDesc.Name), TableName: parser.Name(tableName), } tableNames = append(tableNames, tn) } return tableNames, nil }
// getKeysForTableDescriptor retrieves the KV keys corresponding // to the zone, name and descriptor of a table. func getKeysForTableDescriptor( tableDesc *sqlbase.TableDescriptor, ) (zoneKey roachpb.Key, nameKey roachpb.Key, descKey roachpb.Key) { zoneKey = sqlbase.MakeZoneKey(tableDesc.ID) nameKey = sqlbase.MakeNameMetadataKey(tableDesc.ParentID, tableDesc.GetName()) descKey = sqlbase.MakeDescMetadataKey(tableDesc.ID) return }
func getKeysForDatabaseDescriptor( dbDesc *sqlbase.DatabaseDescriptor, ) (zoneKey roachpb.Key, nameKey roachpb.Key, descKey roachpb.Key) { zoneKey = sqlbase.MakeZoneKey(dbDesc.ID) nameKey = sqlbase.MakeNameMetadataKey(keys.RootNamespaceID, dbDesc.GetName()) descKey = sqlbase.MakeDescMetadataKey(dbDesc.ID) return }
// ShowDatabases returns all the databases. // Privileges: None. // Notes: postgres does not have a "show databases" // mysql has a "SHOW DATABASES" permission, but we have no system-level permissions. func (p *planner) ShowDatabases(n *parser.ShowDatabases) (planNode, error) { // TODO(pmattis): This could be implemented as: // // SELECT id FROM system.namespace WHERE parentID = 0 prefix := sqlbase.MakeNameMetadataKey(keys.RootNamespaceID, "") sr, err := p.txn.Scan(prefix, prefix.PrefixEnd(), 0) if err != nil { return nil, err } v := &valuesNode{columns: []ResultColumn{{Name: "Database", Typ: parser.TypeString}}} for _, row := range sr { _, name, err := encoding.DecodeUnsafeStringAscending( bytes.TrimPrefix(row.Key, prefix), nil) if err != nil { return nil, err } v.rows = append(v.rows, []parser.Datum{parser.NewDString(name)}) } return v, nil }
func TestDropTable(t *testing.T) { defer leaktest.AfterTest(t)() s, sqlDB, kvDB := setup(t) defer cleanup(s, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.kv (k CHAR PRIMARY KEY, v CHAR); INSERT INTO t.kv VALUES ('c', 'e'), ('a', 'c'), ('b', 'd'); `); err != nil { t.Fatal(err) } nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "kv") gr, pErr := kvDB.Get(nameKey) if pErr != nil { t.Fatal(pErr) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if pErr := kvDB.GetProto(descKey, desc); pErr != nil { t.Fatal(pErr) } tableDesc := desc.GetTable() // Add a zone config for the table. cfg := config.DefaultZoneConfig() buf, err := protoutil.Marshal(&cfg) if err != nil { t.Fatal(err) } if _, err := sqlDB.Exec(`INSERT INTO system.zones VALUES ($1, $2)`, tableDesc.ID, buf); err != nil { t.Fatal(err) } zoneKey := sqlbase.MakeZoneKey(tableDesc.ID) if gr, err := kvDB.Get(zoneKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatalf("zone config entry not found") } tablePrefix := keys.MakeTablePrefix(uint32(tableDesc.ID)) tableStartKey := roachpb.Key(tablePrefix) tableEndKey := tableStartKey.PrefixEnd() if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil { t.Fatal(err) } else if l := 6; len(kvs) != l { t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs)) } if _, err := sqlDB.Exec(`DROP TABLE t.kv`); err != nil { t.Fatal(err) } // Test that deleted table cannot be used. This prevents regressions where // name -> descriptor ID caches might make this statement erronously work. if _, err := sqlDB.Exec(`SELECT * FROM t.kv`); !testutils.IsError(err, `table "t.kv" does not exist`) { t.Fatalf("different error than expected: %s", err) } if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil { t.Fatal(err) } else if l := 0; len(kvs) != l { t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs)) } if gr, err := kvDB.Get(descKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("table descriptor still exists after the table is dropped") } if gr, err := kvDB.Get(nameKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("table namekey still exists after the table is dropped") } if gr, err := kvDB.Get(zoneKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("zone config entry still exists after the table is dropped") } }
func TestDropIndex(t *testing.T) { defer leaktest.AfterTest(t)() s, sqlDB, kvDB := setup(t) defer cleanup(s, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.kv (k CHAR PRIMARY KEY, v CHAR); CREATE INDEX foo on t.kv (v); INSERT INTO t.kv VALUES ('c', 'e'), ('a', 'c'), ('b', 'd'); `); err != nil { t.Fatal(err) } nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "kv") gr, pErr := kvDB.Get(nameKey) if pErr != nil { t.Fatal(pErr) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if pErr := kvDB.GetProto(descKey, desc); pErr != nil { t.Fatal(pErr) } tableDesc := desc.GetTable() status, i, err := tableDesc.FindIndexByName("foo") if err != nil { t.Fatal(err) } if status != sqlbase.DescriptorActive { t.Fatal("Index 'foo' is not active.") } indexPrefix := sqlbase.MakeIndexKeyPrefix(tableDesc.ID, tableDesc.Indexes[i].ID) indexStartKey := roachpb.Key(indexPrefix) indexEndKey := indexStartKey.PrefixEnd() if kvs, err := kvDB.Scan(indexStartKey, indexEndKey, 0); err != nil { t.Fatal(err) } else if l := 3; len(kvs) != l { t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs)) } if _, err := sqlDB.Exec(`DROP INDEX t.kv@foo`); err != nil { t.Fatal(err) } if kvs, err := kvDB.Scan(indexStartKey, indexEndKey, 0); err != nil { t.Fatal(err) } else if l := 0; len(kvs) != l { t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs)) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } tableDesc = desc.GetTable() if _, _, err := tableDesc.FindIndexByName("foo"); err == nil { t.Fatalf("table descriptor still contains index after index is dropped") } if err != nil { t.Fatal(err) } }
func TestCommandsWithPendingMutations(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (a CHAR PRIMARY KEY, b CHAR, c CHAR, INDEX foo (c)); `); err != nil { t.Fatal(err) } // Read table descriptor nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mt := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } // Test CREATE INDEX in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`CREATE INDEX foo ON t.test (c)`); !testutils.IsError(err, `index "foo" being dropped, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`CREATE INDEX foo ON t.test (c)`); !testutils.IsError(err, `duplicate index name: "foo"`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Add column DROP mutation "b" mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`CREATE INDEX bar ON t.test (b)`); !testutils.IsError(err, `index "bar" contains unknown column "b"`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) // An index referencing a column mutation that is being added // is allowed to be added. if _, err := sqlDB.Exec(`CREATE INDEX bar ON t.test (b)`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test DROP INDEX in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) // Noop. if _, err := sqlDB.Exec(`DROP INDEX t.test@foo`); err != nil { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`DROP INDEX t.test@foo`); !testutils.IsError(err, `index "foo" in the middle of being added, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Test ALTER TABLE ADD/DROP column in the presence of mutations. // Add column DROP mutation "b" mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD b CHAR`); !testutils.IsError(err, `column "b" being dropped, try again later`) { t.Fatal(err) } // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP b`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD b CHAR`); !testutils.IsError(err, `duplicate column name: "b"`) { t.Fatal(err) } if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP b`); !testutils.IsError(err, `column "b" in the middle of being added, try again later`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test ALTER TABLE ADD CONSTRAINT in the presence of mutations. // Add index DROP mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT foo UNIQUE (c)`); !testutils.IsError(err, `index "foo" being dropped, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT foo UNIQUE (c)`); !testutils.IsError(err, `duplicate index name: "foo"`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Add column mutation "b" mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT bar UNIQUE (b)`); !testutils.IsError(err, `index "bar" contains unknown column "b"`) { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // "b" is being added. mt.writeColumnMutation("b", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test ADD CONSTRAINT bar UNIQUE (b)`); err != nil { t.Fatal(err) } // Make "b" live. mt.makeMutationsActive() // Test DROP CONSTRAINT in the presence of mutations. // Add index mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) // Noop. if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP CONSTRAINT foo`); err != nil { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // "foo" is being added. mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test DROP CONSTRAINT foo`); !testutils.IsError(err, `constraint "foo" in the middle of being added, try again later`) { t.Fatal(err) } // Make "foo" live. mt.makeMutationsActive() // Rename column/index, while index is under mutation. // Add index mutation "foo"" mt.writeIndexMutation("foo", sqlbase.DescriptorMutation{}) if _, err := sqlDB.Exec(`ALTER INDEX t.test@foo RENAME to ufo`); err != nil { mt.Fatal(err) } if _, err := sqlDB.Exec(`ALTER TABLE t.test RENAME COLUMN c TO d`); err != nil { mt.Fatal(err) } // The mutation in the table descriptor has changed and we would like // to update our copy to make it live. if err := mt.kvDB.GetProto(mt.descKey, mt.desc); err != nil { mt.Fatal(err) } // Make "ufo" live. mt.makeMutationsActive() // The index has been renamed to ufo, and the column to d. _ = mt.checkQueryResponse("SHOW INDEXES FROM t.test", [][]string{{"test", "primary", "true", "1", "a", "ASC", "false"}, {"test", "ufo", "false", "1", "d", "ASC", "false"}}) // Rename column under mutation works properly. // Add column mutation "b". mt.writeColumnMutation("b", sqlbase.DescriptorMutation{}) if _, err := sqlDB.Exec(`ALTER TABLE t.test RENAME COLUMN b TO e`); err != nil { mt.Fatal(err) } // The mutation in the table descriptor has changed and we would like // to update our copy to make it live. if err := mt.kvDB.GetProto(mt.descKey, mt.desc); err != nil { mt.Fatal(err) } // Make column "e" live. mt.makeMutationsActive() // Column b changed to d. _ = mt.checkQueryResponse("SHOW COLUMNS FROM t.test", [][]string{{"a", "STRING", "false", "NULL"}, {"d", "STRING", "true", "NULL"}, {"e", "STRING", "true", "NULL"}}) // Try to change column defaults while column is under mutation. mt.writeColumnMutation("e", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_ADD}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ALTER COLUMN e SET DEFAULT 'a'`); !testutils.IsError( err, `column "e" in the middle of being added`) { t.Fatal(err) } mt.makeMutationsActive() mt.writeColumnMutation("e", sqlbase.DescriptorMutation{Direction: sqlbase.DescriptorMutation_DROP}) if _, err := sqlDB.Exec(`ALTER TABLE t.test ALTER COLUMN e SET DEFAULT 'a'`); !testutils.IsError( err, `column "e" in the middle of being dropped`) { t.Fatal(err) } mt.makeMutationsActive() }
// TestOperationsWithUniqueColumnMutation tests all the operations while an // index mutation refers to a column mutation. func TestOperationsWithUniqueColumnMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) // Create a table with column i and an index on v and i. if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, i CHAR, INDEX foo (i, v)); `); err != nil { t.Fatal(err) } // read table descriptor nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } starQuery := `SELECT * FROM t.test` indexQuery := `SELECT i FROM t.test@foo` // Run the tests for both states. for _, state := range []sqlbase.DescriptorMutation_State{ sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY, } { // Init table to start state. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z", "q"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2, $3)`, row[0], row[1], row[2]); err != nil { t.Fatal(err) } } // Check that the table only contains the initRows. _ = mTest.checkQueryResponse(starQuery, initRows) // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Insert a row into the table. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Error(err) } // Make column "i" and index "foo" live. mTest.makeMutationsActive() // column "i" has no entry. _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z", "q"}, {"c", "x", "NULL"}}) if state == sqlbase.DescriptorMutation_DELETE_ONLY { // No index entry for row "c" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"q"}}) } else { // Index entry for row "c" _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}}) } // Add index "foo" as a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Updating column "i" for a row fails. if _, err := sqlDB.Exec(`UPDATE t.test SET (v, i) = ('u', 'u') WHERE k = 'a'`); !testutils.IsError(err, `column "i" does not exist`) { t.Error(err) } // TODO(vivek): Fix #6691. // Update a row without specifying mutation column "i". //if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'u' WHERE k = 'a'`); err != nil { // t.Error(err) //} // Make column "i" and index "foo" live. mTest.makeMutationsActive() // The update to column "v" is seen; there is no effect on column "i". //_ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "u", "q"}, {"c", "x", "NULL"}}) // No change in index "foo" //if state == sqlbase.DescriptorMutation_DELETE_ONLY { // _ = mTest.checkQueryResponse(indexQuery, [][]string{{"q"}}) //} else { // _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}, {"q"}}) //} // Add index "foo" as a mutation. //mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Make column "i" a mutation. //mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Delete row "a". //if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'a'`); err != nil { // t.Error(err) //} // Make column "i" and index "foo" live. // mTest.makeMutationsActive() // Row "a" is deleted. numVals is the number of non-NULL values seen, // or the number of KV values belonging to all the rows in the table // excluding row "a" since it's deleted. //numVals := mTest.checkQueryResponse(starQuery, [][]string{{"c", "x", "NULL"}}) //if state == sqlbase.DescriptorMutation_DELETE_ONLY { // _ = mTest.checkQueryResponse(indexQuery, [][]string{}) //} else { // _ = mTest.checkQueryResponse(indexQuery, [][]string{{"NULL"}}) //} // Check that there are no hidden KV values for row "a", // and column "i" for row "a" was deleted. //mTest.checkTableSize(numVals) } }
func TestOperationsWithIndexMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo (v)); `); err != nil { t.Fatal(err) } // read table descriptor nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } starQuery := `SELECT * FROM t.test` indexQuery := `SELECT v FROM t.test@foo` // See the effect of the operations depending on the state. for _, state := range []sqlbase.DescriptorMutation_State{sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY} { // Init table with some entries. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z"}, {"b", "y"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2)`, row[0], row[1]); err != nil { t.Fatal(err) } } _ = mTest.checkQueryResponse(starQuery, initRows) // Index foo is visible. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) // Index foo is invisible once it's a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) if _, err := sqlDB.Query(indexQuery); !testutils.IsError(err, `index "foo" not found`) { t.Fatal(err) } // Insert a new entry. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"b", "y"}, {"c", "x"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() if state == sqlbase.DescriptorMutation_DELETE_ONLY { // "x" didn't get added to the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) } else { // "x" got added to the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"x"}, {"y"}, {"z"}}) } // Make "foo" a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Update. if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'w' WHERE k = 'c'`); err != nil { t.Fatal(err) } // Update "v" to its current value "z" in row "a". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'z' WHERE k = 'a'`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"b", "y"}, {"c", "w"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() if state == sqlbase.DescriptorMutation_DELETE_ONLY { // updating "x" -> "w" is a noop on the index, // updating "z" -> "z" results in "z" being deleted from the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"y"}, {"z"}}) } else { // updating "x" -> "w" results in the index updating from "x" -> "w", // updating "z" -> "z" is a noop on the index. _ = mTest.checkQueryResponse(indexQuery, [][]string{{"w"}, {"y"}, {"z"}}) } // Make "foo" a mutation. mTest.writeIndexMutation("foo", sqlbase.DescriptorMutation{State: state}) // Delete row "b". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'b'`); err != nil { t.Fatal(err) } _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}, {"c", "w"}}) // Make index "foo" live so that we can read it. mTest.makeMutationsActive() // Deleting row "b" deletes "y" from the index. if state == sqlbase.DescriptorMutation_DELETE_ONLY { mTest.checkQueryResponse(indexQuery, [][]string{{"z"}}) } else { mTest.checkQueryResponse(indexQuery, [][]string{{"w"}, {"z"}}) } } // Check that a mutation can only be inserted with an explicit mutation state. tableDesc := desc.GetTable() tableDesc.Mutations = []sqlbase.DescriptorMutation{{Descriptor_: &sqlbase.DescriptorMutation_Index{Index: &tableDesc.Indexes[len(tableDesc.Indexes)-1]}}} tableDesc.Indexes = tableDesc.Indexes[:len(tableDesc.Indexes)-1] if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, index foo, id 2") { t.Fatal(err) } }
func TestOperationsWithColumnMutation(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, i CHAR DEFAULT 'i'); `); err != nil { t.Fatal(err) } // read table descriptor nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } starQuery := `SELECT * FROM t.test` // Run the tests for both states. for _, state := range []sqlbase.DescriptorMutation_State{sqlbase.DescriptorMutation_DELETE_ONLY, sqlbase.DescriptorMutation_WRITE_ONLY} { // Init table to start state. if _, err := sqlDB.Exec(`TRUNCATE TABLE t.test`); err != nil { t.Fatal(err) } initRows := [][]string{{"a", "z", "q"}} for _, row := range initRows { if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ($1, $2, $3)`, row[0], row[1], row[2]); err != nil { t.Fatal(err) } } // Check that the table only contains the initRows. _ = mTest.checkQueryResponse(starQuery, initRows) // Add column "i" as a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // A direct read of column "i" fails. if _, err := sqlDB.Query(`SELECT i FROM t.test`); err == nil { t.Fatalf("Read succeeded despite column being in %v state", sqlbase.DescriptorMutation{State: state}) } // The table only contains columns "k" and "v". _ = mTest.checkQueryResponse(starQuery, [][]string{{"a", "z"}}) // The column backfill uses Put instead of CPut because it depends on // an INSERT of a column in the WRITE_ONLY state failing. These two // tests guarantee that. // Inserting a row into the table while specifying column "i" results in an error. if _, err := sqlDB.Exec(`INSERT INTO t.test (k, v, i) VALUES ('b', 'y', 'i')`); !testutils.IsError(err, `column "i" does not exist`) { t.Fatal(err) } // Repeating the same without specifying the columns results in a different error. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('b', 'y', 'i')`); !testutils.IsError(err, "INSERT has more expressions than target columns: 3/2") { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Check that we can read all the rows and columns. _ = mTest.checkQueryResponse(starQuery, initRows) var afterInsert, afterUpdate, afterDelete [][]string if state == sqlbase.DescriptorMutation_DELETE_ONLY { // The default value of "i" for column "i" is not written. afterInsert = [][]string{{"a", "z", "q"}, {"c", "x", "NULL"}} // Update is a noop for column "i". afterUpdate = [][]string{{"a", "u", "q"}, {"c", "x", "NULL"}} // Delete also deletes column "i". afterDelete = [][]string{{"c", "x", "NULL"}} } else { // The default value of "i" for column "i" is written. afterInsert = [][]string{{"a", "z", "q"}, {"c", "x", "i"}} // Update is a noop for column "i". afterUpdate = [][]string{{"a", "u", "q"}, {"c", "x", "i"}} // Delete also deletes column "i". afterDelete = [][]string{{"c", "x", "i"}} } // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Insert a row into the table. if _, err := sqlDB.Exec(`INSERT INTO t.test VALUES ('c', 'x')`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Notice that the default value of "i" is only written when the // descriptor is in the WRITE_ONLY state. _ = mTest.checkQueryResponse(starQuery, afterInsert) // The column backfill uses Put instead of CPut because it depends on // an UPDATE of a column in the WRITE_ONLY state failing. This test // guarantees that. // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Updating column "i" for a row fails. if _, err := sqlDB.Exec(`UPDATE t.test SET (v, i) = ('u', 'u') WHERE k = 'a'`); !testutils.IsError(err, `column "i" does not exist`) { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // The above failed update was a noop. _ = mTest.checkQueryResponse(starQuery, afterInsert) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Update a row without specifying mutation column "i". if _, err := sqlDB.Exec(`UPDATE t.test SET v = 'u' WHERE k = 'a'`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // The update to column "v" is seen; there is no effect on column "i". _ = mTest.checkQueryResponse(starQuery, afterUpdate) // Make column "i" a mutation. mTest.writeColumnMutation("i", sqlbase.DescriptorMutation{State: state}) // Delete row "a". if _, err := sqlDB.Exec(`DELETE FROM t.test WHERE k = 'a'`); err != nil { t.Fatal(err) } // Make column "i" live so that it is read. mTest.makeMutationsActive() // Row "a" is deleted. numVals is the number of non-NULL values seen, // or the number of KV values belonging to all the rows in the table // excluding row "a" since it's deleted. numVals := mTest.checkQueryResponse(starQuery, afterDelete) // Check that there are no hidden KV values for row "a", // and column "i" for row "a" was deleted. mTest.checkTableSize(numVals) } // Check that a mutation can only be inserted with an explicit mutation state, and direction. tableDesc := desc.GetTable() tableDesc.Mutations = []sqlbase.DescriptorMutation{{}} if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, and no column/index descriptor") { t.Fatal(err) } tableDesc.Mutations = []sqlbase.DescriptorMutation{{Descriptor_: &sqlbase.DescriptorMutation_Column{Column: &tableDesc.Columns[len(tableDesc.Columns)-1]}}} tableDesc.Columns = tableDesc.Columns[:len(tableDesc.Columns)-1] if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction NONE, col i, id 3") { t.Fatal(err) } tableDesc.Mutations[0].State = sqlbase.DescriptorMutation_DELETE_ONLY if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state DELETE_ONLY, direction NONE, col i, id 3") { t.Fatal(err) } tableDesc.Mutations[0].State = sqlbase.DescriptorMutation_UNKNOWN tableDesc.Mutations[0].Direction = sqlbase.DescriptorMutation_DROP if err := tableDesc.Validate(); !testutils.IsError(err, "mutation in state UNKNOWN, direction DROP, col i, id 3") { t.Fatal(err) } }
func (tk tableKey) Key() roachpb.Key { return sqlbase.MakeNameMetadataKey(tk.parentID, tk.name) }
func TestDropDatabase(t *testing.T) { defer leaktest.AfterTest(t)() s, sqlDB, kvDB := setup(t) defer cleanup(s, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.kv (k CHAR PRIMARY KEY, v CHAR); INSERT INTO t.kv VALUES ('c', 'e'), ('a', 'c'), ('b', 'd'); `); err != nil { t.Fatal(err) } dbNameKey := sqlbase.MakeNameMetadataKey(keys.RootNamespaceID, "t") r, pErr := kvDB.Get(dbNameKey) if pErr != nil { t.Fatal(pErr) } if !r.Exists() { t.Fatalf(`database "t" does not exist`) } dbDescKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(r.ValueInt())) desc := &sqlbase.Descriptor{} if pErr := kvDB.GetProto(dbDescKey, desc); pErr != nil { t.Fatal(pErr) } dbDesc := desc.GetDatabase() tbNameKey := sqlbase.MakeNameMetadataKey(dbDesc.ID, "kv") gr, pErr := kvDB.Get(tbNameKey) if pErr != nil { t.Fatal(pErr) } if !gr.Exists() { t.Fatalf(`table "kv" does not exist`) } tbDescKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) if pErr := kvDB.GetProto(tbDescKey, desc); pErr != nil { t.Fatal(pErr) } tbDesc := desc.GetTable() // Add a zone config for both the table and database. cfg := config.DefaultZoneConfig() buf, err := protoutil.Marshal(&cfg) if err != nil { t.Fatal(err) } if _, err := sqlDB.Exec(`INSERT INTO system.zones VALUES ($1, $2)`, tbDesc.ID, buf); err != nil { t.Fatal(err) } if _, err := sqlDB.Exec(`INSERT INTO system.zones VALUES ($1, $2)`, dbDesc.ID, buf); err != nil { t.Fatal(err) } tbZoneKey := sqlbase.MakeZoneKey(tbDesc.ID) dbZoneKey := sqlbase.MakeZoneKey(dbDesc.ID) if gr, err := kvDB.Get(tbZoneKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatalf("table zone config entry not found") } if gr, err := kvDB.Get(dbZoneKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatalf("database zone config entry not found") } tablePrefix := keys.MakeTablePrefix(uint32(tbDesc.ID)) tableStartKey := roachpb.Key(tablePrefix) tableEndKey := tableStartKey.PrefixEnd() if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil { t.Fatal(err) } else if l := 6; len(kvs) != l { t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs)) } if _, err := sqlDB.Exec(`DROP DATABASE t`); err != nil { t.Fatal(err) } if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil { t.Fatal(err) } else if l := 0; len(kvs) != l { t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs)) } if gr, err := kvDB.Get(tbDescKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("table descriptor still exists after database is dropped: %q", tbDescKey) } if gr, err := kvDB.Get(tbNameKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("table descriptor key still exists after database is dropped") } if gr, err := kvDB.Get(dbDescKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("database descriptor still exists after database is dropped") } if gr, err := kvDB.Get(dbNameKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("database descriptor key still exists after database is dropped") } if gr, err := kvDB.Get(tbZoneKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("table zone config entry still exists after the database is dropped") } if gr, err := kvDB.Get(dbZoneKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatalf("database zone config entry still exists after the database is dropped") } }
// Test schema change purge failure doesn't leave DB in a bad state. func TestSchemaChangePurgeFailure(t *testing.T) { defer leaktest.AfterTest(t)() // Disable the async schema changer. var enableAsyncSchemaChanges uint32 attempts := 0 ctx, _ := createTestServerContext() ctx.TestingKnobs = base.TestingKnobs{ SQLExecutor: &csql.ExecutorTestingKnobs{ SchemaChangersStartBackfillNotification: func() error { attempts++ // Return a deadline exceeded error during the second attempt // which attempts to clean up the schema change. if attempts == 2 { return errors.New("context deadline exceeded") } return nil }, }, SQLSchemaChangeManager: &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: func() error { if enable := atomic.LoadUint32(&enableAsyncSchemaChanges); enable == 0 { return errors.New("async schema changes are disabled") } return nil }, // Speed up evaluation of async schema changes so that it // processes a purged schema change quickly. AsyncSchemaChangerExecQuickly: true, }, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k INT PRIMARY KEY, v INT); `); err != nil { t.Fatal(err) } // Bulk insert. maxValue := csql.IndexBackfillChunkSize + 1 insert := fmt.Sprintf(`INSERT INTO t.test VALUES (%d, %d)`, 0, maxValue) for i := 1; i <= maxValue; i++ { insert += fmt.Sprintf(` ,(%d, %d)`, i, maxValue-i) } if _, err := sqlDB.Exec(insert); err != nil { t.Fatal(err) } // Add a row with a duplicate value for v if _, err := sqlDB.Exec( `INSERT INTO t.test VALUES ($1, $2)`, maxValue+1, maxValue, ); err != nil { t.Fatal(err) } // A schema change that violates integrity constraints. if _, err := sqlDB.Exec( "CREATE UNIQUE INDEX foo ON t.test (v)", ); !testutils.IsError(err, "violates unique constraint") { t.Fatal(err) } // The deadline exceeded error in the schema change purge results in no // retry attempts of the purge. if e := 2; attempts != e { t.Fatalf("%d retries, despite allowing only (schema change + reverse) = %d", attempts, e) } // The index doesn't exist if _, err := sqlDB.Query( `SELECT v from t.test@foo`, ); !testutils.IsError(err, "index .* not found") { t.Fatal(err) } // Read table descriptor. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } tableDesc := desc.GetTable() // There is still a mutation hanging off of it. if e := 1; len(tableDesc.Mutations) != e { t.Fatalf("the table has %d instead of %d mutations", len(tableDesc.Mutations), e) } // The mutation is for a DROP. if tableDesc.Mutations[0].Direction != sqlbase.DescriptorMutation_DROP { t.Fatalf("the table has mutation %v instead of a DROP", tableDesc.Mutations[0]) } // There is still some garbage index data that needs to be purged. All the // rows from k = 0 to k = maxValue have index values. The k = maxValue + 1 // row with the conflict doesn't contain an index value. numGarbageValues := csql.IndexBackfillChunkSize tablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(tableDesc.ID))) tableEnd := tablePrefix.PrefixEnd() if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil { t.Fatal(err) } else if e := 2*(maxValue+2) + numGarbageValues; len(kvs) != e { t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs)) } // Enable async schema change processing to ensure that it cleans up the // above garbage left behind. atomic.StoreUint32(&enableAsyncSchemaChanges, 1) util.SucceedsSoon(t, func() error { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } tableDesc := desc.GetTable() if len(tableDesc.Mutations) > 0 { return util.Errorf("%d mutations remaining", len(tableDesc.Mutations)) } return nil }) // No garbage left behind. numGarbageValues = 0 if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil { t.Fatal(err) } else if e := 2*(maxValue+2) + numGarbageValues; len(kvs) != e { t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs)) } }
// Test schema change backfills are not affected by various operations // that run simultaneously. func TestRaceWithBackfill(t *testing.T) { defer leaktest.AfterTest(t)() // Disable asynchronous schema change execution to allow synchronous path // to trigger start of backfill notification. var backfillNotification chan bool ctx, _ := createTestServerContext() ctx.TestingKnobs = base.TestingKnobs{ SQLExecutor: &csql.ExecutorTestingKnobs{ SchemaChangersStartBackfillNotification: func() error { if backfillNotification != nil { // Close channel to notify that the backfill has started. close(backfillNotification) } return nil }, }, SQLSchemaChangeManager: &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, }, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k INT PRIMARY KEY, v INT, pi DECIMAL DEFAULT (DECIMAL '3.14')); CREATE UNIQUE INDEX vidx ON t.test (v); `); err != nil { t.Fatal(err) } // Bulk insert. maxValue := 4000 insert := fmt.Sprintf(`INSERT INTO t.test VALUES (%d, %d)`, 0, maxValue) for i := 1; i <= maxValue; i++ { insert += fmt.Sprintf(` ,(%d, %d)`, i, maxValue-i) } if _, err := sqlDB.Exec(insert); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } tableDesc := desc.GetTable() tablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(tableDesc.ID))) tableEnd := tablePrefix.PrefixEnd() // number of keys == 4 * number of rows; 3 columns and 1 index entry for // each row. if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil { t.Fatal(err) } else if e := 4 * (maxValue + 1); len(kvs) != e { t.Fatalf("expected %d key value pairs, but got %d", e, len(kvs)) } // Run some schema changes with operations. // Add column. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "ALTER TABLE t.test ADD COLUMN x DECIMAL DEFAULT (DECIMAL '1.4')", maxValue, 5, descKey, backfillNotification) // Drop column. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "ALTER TABLE t.test DROP pi", maxValue, 4, descKey, backfillNotification) // Add index. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "CREATE UNIQUE INDEX foo ON t.test (v)", maxValue, 5, descKey, backfillNotification) // Drop index. backfillNotification = make(chan bool) runSchemaChangeWithOperations( t, sqlDB, kvDB, "DROP INDEX t.test@vidx", maxValue, 4, descKey, backfillNotification) // Verify that the index foo over v is consistent, and that column x has // been backfilled properly. rows, err := sqlDB.Query(`SELECT v, x from t.test@foo`) if err != nil { t.Fatal(err) } count := 0 for ; rows.Next(); count++ { var val int var x float64 if err := rows.Scan(&val, &x); err != nil { t.Errorf("row %d scan failed: %s", count, err) continue } if count != val { t.Errorf("e = %d, v = %d", count, val) } if 1.4 != x { t.Errorf("e = %f, v = %f", 1.4, x) } } if err := rows.Err(); err != nil { t.Fatal(err) } eCount := maxValue + 1 if eCount != count { t.Fatalf("read the wrong number of rows: e = %d, v = %d", eCount, count) } // Verify that a table delete in the middle of a backfill works properly. // The backfill will terminate in the middle, and the delete will // successfully delete all the table data. // // This test could be made its own test but is placed here to speed up the // testing. backfillNotification = make(chan bool) // Run the schema change in a separate goroutine. var wg sync.WaitGroup wg.Add(1) go func() { // Start schema change that eventually runs a backfill. if _, err := sqlDB.Exec("CREATE UNIQUE INDEX bar ON t.test (v)"); err != nil { t.Error(err) } wg.Done() }() // Wait until the schema change backfill starts. <-backfillNotification // Wait for a short bit to ensure that the backfill has likely progressed // and written some data, but not long enough that the backfill has // completed. time.Sleep(10 * time.Millisecond) if _, err := sqlDB.Exec("DROP TABLE t.test"); err != nil { t.Fatal(err) } // Wait until the schema change is done. wg.Wait() // Ensure that the table data has been deleted. if kvs, err := kvDB.Scan(tablePrefix, tableEnd, 0); err != nil { t.Fatal(err) } else if len(kvs) != 0 { t.Fatalf("expected %d key value pairs, but got %d", 0, len(kvs)) } }
func TestAsyncSchemaChanger(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable synchronous schema change execution so the asynchronous schema // changer executes all schema changes. ctx, _ := createTestServerContext() ctx.TestingKnobs = base.TestingKnobs{ SQLExecutor: &csql.ExecutorTestingKnobs{ SyncSchemaChangersFilter: func(tscc csql.TestingSchemaChangerCollection) { tscc.ClearSchemaChangers() }, }, SQLSchemaChangeManager: &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecQuickly: true, }, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } // A long running schema change operation runs through // a state machine that increments the version by 3. expectedVersion := desc.GetTable().Version + 3 // Run some schema change if _, err := sqlDB.Exec(` CREATE INDEX foo ON t.test (v) `); err != nil { t.Fatal(err) } retryOpts := retry.Options{ InitialBackoff: 20 * time.Millisecond, MaxBackoff: 200 * time.Millisecond, Multiplier: 2, } // Wait until index is created. for r := retry.Start(retryOpts); r.Next(); { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } if len(desc.GetTable().Indexes) == 1 { break } } // Ensure that the indexes have been created. mTest := mutationTest{ T: t, kvDB: kvDB, sqlDB: sqlDB, descKey: descKey, desc: desc, } indexQuery := `SELECT v FROM t.test@foo` _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) // Ensure that the version has been incremented. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion := desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Apply a schema change that only sets the UpVersion bit. expectedVersion = newVersion + 1 if _, err := sqlDB.Exec(` ALTER INDEX t.test@foo RENAME TO ufo `); err != nil { t.Fatal(err) } for r := retry.Start(retryOpts); r.Next(); { // Ensure that the version gets incremented. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } name := desc.GetTable().Indexes[0].Name if name != "ufo" { t.Fatalf("bad index name %s", name) } newVersion = desc.GetTable().Version if newVersion == expectedVersion { break } } // Run many schema changes simultaneously and check // that they all get executed. count := 5 for i := 0; i < count; i++ { cmd := fmt.Sprintf(`CREATE INDEX foo%d ON t.test (v)`, i) if _, err := sqlDB.Exec(cmd); err != nil { t.Fatal(err) } } // Wait until indexes are created. for r := retry.Start(retryOpts); r.Next(); { if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } if len(desc.GetTable().Indexes) == count+1 { break } } for i := 0; i < count; i++ { indexQuery := fmt.Sprintf(`SELECT v FROM t.test@foo%d`, i) _ = mTest.checkQueryResponse(indexQuery, [][]string{{"b"}, {"d"}}) } }
func TestSchemaChangeProcess(t *testing.T) { defer leaktest.AfterTest(t)() // The descriptor changes made must have an immediate effect // so disable leases on tables. defer csql.TestDisableTableLeases()() // Disable external processing of mutations. ctx, _ := createTestServerContext() ctx.TestingKnobs.SQLSchemaChangeManager = &csql.SchemaChangeManagerTestingKnobs{ AsyncSchemaChangerExecNotification: schemaChangeManagerDisabled, } server, sqlDB, kvDB := setupWithContext(t, &ctx) defer cleanup(server, sqlDB) var id = sqlbase.ID(keys.MaxReservedDescID + 2) var node = roachpb.NodeID(2) db := server.DB() leaseMgr := csql.NewLeaseManager(0, *db, hlc.NewClock(hlc.UnixNano), csql.LeaseManagerTestingKnobs{}) changer := csql.NewSchemaChangerForTesting(id, 0, node, *db, leaseMgr) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR, INDEX foo(v)); INSERT INTO t.test VALUES ('a', 'b'), ('c', 'd'); `); err != nil { t.Fatal(err) } // Read table descriptor for version. nameKey := sqlbase.MakeNameMetadataKey(keys.MaxReservedDescID+1, "test") gr, err := kvDB.Get(nameKey) if err != nil { t.Fatal(err) } if !gr.Exists() { t.Fatalf("Name entry %q does not exist", nameKey) } descKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(gr.ValueInt())) desc := &sqlbase.Descriptor{} // Check that MaybeIncrementVersion doesn't increment the version // when the up_version bit is not set. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } expectedVersion := desc.GetTable().Version desc, err = changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } newVersion := desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } isDone, err := changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", desc.GetTable()) } // Check that MaybeIncrementVersion increments the version // correctly. expectedVersion++ desc.GetTable().UpVersion = true if err := kvDB.Put(descKey, desc); err != nil { t.Fatal(err) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } desc, err = changer.MaybeIncrementVersion() if err != nil { t.Fatal(err) } savedDesc := &sqlbase.Descriptor{} if err := kvDB.GetProto(descKey, savedDesc); err != nil { t.Fatal(err) } newVersion = desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version in returned desc; e = %d, v = %d", expectedVersion, newVersion) } newVersion = savedDesc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version in saved desc; e = %d, v = %d", expectedVersion, newVersion) } isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if !isDone { t.Fatalf("table expected to not have an outstanding schema change: %v", desc.GetTable()) } // Check that RunStateMachineBeforeBackfill doesn't do anything // if there are no mutations queued. if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } newVersion = desc.GetTable().Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } // Check that RunStateMachineBeforeBackfill functions properly. if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } table := desc.GetTable() expectedVersion = table.Version // Make a copy of the index for use in a mutation. index := protoutil.Clone(&table.Indexes[0]).(*sqlbase.IndexDescriptor) index.Name = "bar" index.ID = table.NextIndexID table.NextIndexID++ changer = csql.NewSchemaChangerForTesting(id, table.NextMutationID, node, *db, leaseMgr) table.Mutations = append(table.Mutations, sqlbase.DescriptorMutation{ Descriptor_: &sqlbase.DescriptorMutation_Index{Index: index}, Direction: sqlbase.DescriptorMutation_ADD, State: sqlbase.DescriptorMutation_DELETE_ONLY, MutationID: table.NextMutationID, }) table.NextMutationID++ // Run state machine in both directions. for _, direction := range []sqlbase.DescriptorMutation_Direction{sqlbase.DescriptorMutation_ADD, sqlbase.DescriptorMutation_DROP} { table.Mutations[0].Direction = direction expectedVersion++ if err := kvDB.Put(descKey, desc); err != nil { t.Fatal(err) } // The expected end state. expectedState := sqlbase.DescriptorMutation_WRITE_ONLY if direction == sqlbase.DescriptorMutation_DROP { expectedState = sqlbase.DescriptorMutation_DELETE_ONLY } // Run two times to ensure idempotency of operations. for i := 0; i < 2; i++ { if err := changer.RunStateMachineBeforeBackfill(); err != nil { t.Fatal(err) } if err := kvDB.GetProto(descKey, desc); err != nil { t.Fatal(err) } table = desc.GetTable() newVersion = table.Version if newVersion != expectedVersion { t.Fatalf("bad version; e = %d, v = %d", expectedVersion, newVersion) } state := table.Mutations[0].State if state != expectedState { t.Fatalf("bad state; e = %d, v = %d", expectedState, state) } } } // RunStateMachineBeforeBackfill() doesn't complete the schema change. isDone, err = changer.IsDone() if err != nil { t.Fatal(err) } if isDone { t.Fatalf("table expected to have an outstanding schema change: %v", desc.GetTable()) } }
func TestDatabaseDescriptor(t *testing.T) { defer leaktest.AfterTest(t)() params, _ := createTestServerParams() s, sqlDB, kvDB := serverutils.StartServer(t, params) defer s.Stopper().Stop() expectedCounter := int64(keys.MaxReservedDescID + 1) // Test values before creating the database. // descriptor ID counter. if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil { t.Fatal(err) } else if actual := ir.ValueInt(); actual != expectedCounter { t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual) } // Database name. nameKey := sqlbase.MakeNameMetadataKey(keys.RootNamespaceID, "test") if gr, err := kvDB.Get(nameKey); err != nil { t.Fatal(err) } else if gr.Exists() { t.Fatal("expected non-existing key") } // Write a descriptor key that will interfere with database creation. dbDescKey := sqlbase.MakeDescMetadataKey(sqlbase.ID(expectedCounter)) dbDesc := &sqlbase.Descriptor{ Union: &sqlbase.Descriptor_Database{ Database: &sqlbase.DatabaseDescriptor{ Name: "sentinel", ID: sqlbase.ID(expectedCounter), Privileges: &sqlbase.PrivilegeDescriptor{}, }, }, } if err := kvDB.CPut(dbDescKey, dbDesc, nil); err != nil { t.Fatal(err) } // Database creation should fail, and nothing should have been written. if _, err := sqlDB.Exec(`CREATE DATABASE test`); !testutils.IsError(err, "unexpected value") { t.Fatalf("unexpected error %v", err) } if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil { t.Fatal(err) } else if actual := ir.ValueInt(); actual != expectedCounter { t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual) } start := roachpb.Key(keys.MakeTablePrefix(uint32(keys.NamespaceTableID))) if kvs, err := kvDB.Scan(start, start.PrefixEnd(), 0); err != nil { t.Fatal(err) } else { if a, e := len(kvs), server.GetBootstrapSchema().SystemDescriptorCount(); a != e { t.Fatalf("expected %d keys to have been written, found %d keys", e, a) } } // Remove the junk; allow database creation to proceed. if err := kvDB.Del(dbDescKey); err != nil { t.Fatal(err) } if _, err := sqlDB.Exec(`CREATE DATABASE test`); err != nil { t.Fatal(err) } expectedCounter++ // Check keys again. // descriptor ID counter. if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil { t.Fatal(err) } else if actual := ir.ValueInt(); actual != expectedCounter { t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual) } // Database name. if gr, err := kvDB.Get(nameKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatal("key is missing") } // database descriptor. if gr, err := kvDB.Get(dbDescKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatal("key is missing") } // Now try to create it again. We should fail, but not increment the counter. if _, err := sqlDB.Exec(`CREATE DATABASE test`); err == nil { t.Fatal("failure expected") } // Check keys again. // descriptor ID counter. if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil { t.Fatal(err) } else if actual := ir.ValueInt(); actual != expectedCounter { t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual) } // Database name. if gr, err := kvDB.Get(nameKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatal("key is missing") } // database descriptor. if gr, err := kvDB.Get(dbDescKey); err != nil { t.Fatal(err) } else if !gr.Exists() { t.Fatal("key is missing") } }
func (dk databaseKey) Key() roachpb.Key { return sqlbase.MakeNameMetadataKey(keys.RootNamespaceID, dk.name) }