// tryLoadSchemaDiffs tries to only load latest schema changes.
// Returns true if the schema is loaded successfully.
// Returns false if the schema can not be loaded by schema diff, then we need to do full load.
func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64) (bool, error) {
if usedVersion == initialVersion || newVersion-usedVersion > maxNumberOfDiffsToLoad {
// If there isn't any used version, or used version is too old, we do full load.
return false, nil
}
if usedVersion > newVersion {
// When user use History Read feature, history schema will be loaded.
// usedVersion may be larger than newVersion, full load is needed.
return false, nil
}
var diffs []*model.SchemaDiff
for usedVersion < newVersion {
usedVersion++
diff, err := m.GetSchemaDiff(usedVersion)
if err != nil {
return false, errors.Trace(err)
}
if diff == nil {
// If diff is missing for any version between used and new version, we fall back to full reload.
return false, nil
}
diffs = append(diffs, diff)
}
builder := infoschema.NewBuilder(do.infoHandle).InitWithOldInfoSchema()
for _, diff := range diffs {
err := builder.ApplyDiff(m, diff)
if err != nil {
return false, errors.Trace(err)
}
}
builder.Build()
return true, nil
}
// Make sure that all tables of infomation_schema could be found in infoschema handle.
func (*testSuite) TestInfoTables(c *C) {
defer testleak.AfterTest(c)()
driver := localstore.Driver{Driver: goleveldb.MemoryDriver{}}
store, err := driver.Open("memory")
c.Assert(err, IsNil)
defer store.Close()
handle, err := infoschema.NewHandle(store)
c.Assert(err, IsNil)
builder, err := infoschema.NewBuilder(handle).InitWithDBInfos(nil, 0)
c.Assert(err, IsNil)
err = builder.Build()
c.Assert(err, IsNil)
is := handle.Get()
c.Assert(is, NotNil)
info_tables := []string{
"SCHEMATA",
"TABLES",
"COLUMNS",
"STATISTICS",
"CHARACTER_SETS",
"COLLATIONS",
"FILES",
"PROFILING",
"PARTITIONS",
"KEY_COLUMN_USAGE",
"REFERENTIAL_CONSTRAINTS",
}
for _, t := range info_tables {
tb, err1 := is.TableByName(model.NewCIStr(infoschema.Name), model.NewCIStr(t))
c.Assert(err1, IsNil)
c.Assert(tb, NotNil)
}
}
// loadInfoSchema loads infoschema at startTS into handle, usedSchemaVersion is the currently used
// infoschema version, if it is the same as the schema version at startTS, we don't need to reload again.
func (do *Domain) loadInfoSchema(handle *infoschema.Handle, usedSchemaVersion int64, startTS uint64) error {
snapshot, err := do.store.GetSnapshot(kv.NewVersion(startTS))
if err != nil {
return errors.Trace(err)
}
m := meta.NewSnapshotMeta(snapshot)
latestSchemaVersion, err := m.GetSchemaVersion()
if err != nil {
return errors.Trace(err)
}
if usedSchemaVersion != 0 && usedSchemaVersion == latestSchemaVersion {
log.Debugf("[ddl] schema version is still %d, no need reload", usedSchemaVersion)
return nil
}
ok, err := do.tryLoadSchemaDiffs(m, usedSchemaVersion, latestSchemaVersion)
if err != nil {
// We can fall back to full load, don't need to return the error.
log.Errorf("[ddl] failed to load schema diff %v", err)
}
if ok {
log.Infof("[ddl] diff load InfoSchema from version %d to %d", usedSchemaVersion, latestSchemaVersion)
return nil
}
schemas, err := do.getAllSchemasWithTablesFromMeta(m)
if err != nil {
return errors.Trace(err)
}
newISBuilder, err := infoschema.NewBuilder(handle).InitWithDBInfos(schemas, latestSchemaVersion)
if err != nil {
return errors.Trace(err)
}
log.Infof("[ddl] full load InfoSchema from version %d to %d", usedSchemaVersion, latestSchemaVersion)
return newISBuilder.Build()
}
// loadInfoSchema loads infoschema at startTS into handle, usedSchemaVersion is the currently used
// infoschema version, if it is the same as the schema version at startTS, we don't need to reload again.
// It returns the latest schema version and an error.
func (do *Domain) loadInfoSchema(handle *infoschema.Handle, usedSchemaVersion int64, startTS uint64) (int64, error) {
snapshot, err := do.store.GetSnapshot(kv.NewVersion(startTS))
if err != nil {
return 0, errors.Trace(err)
}
m := meta.NewSnapshotMeta(snapshot)
latestSchemaVersion, err := m.GetSchemaVersion()
if err != nil {
return 0, errors.Trace(err)
}
if usedSchemaVersion != 0 && usedSchemaVersion == latestSchemaVersion {
return latestSchemaVersion, nil
}
startTime := time.Now()
ok, err := do.tryLoadSchemaDiffs(m, usedSchemaVersion, latestSchemaVersion)
if err != nil {
// We can fall back to full load, don't need to return the error.
log.Errorf("[ddl] failed to load schema diff err %v", err)
}
if ok {
log.Infof("[ddl] diff load InfoSchema from version %d to %d, in %v",
usedSchemaVersion, latestSchemaVersion, time.Since(startTime))
return latestSchemaVersion, nil
}
schemas, err := do.fetchAllSchemasWithTables(m)
if err != nil {
return 0, errors.Trace(err)
}
newISBuilder, err := infoschema.NewBuilder(handle).InitWithDBInfos(schemas, latestSchemaVersion)
if err != nil {
return 0, errors.Trace(err)
}
log.Infof("[ddl] full load InfoSchema from version %d to %d, in %v",
usedSchemaVersion, latestSchemaVersion, time.Since(startTime))
newISBuilder.Build()
return latestSchemaVersion, nil
}
func (*testSuite) TestT(c *C) {
defer testleak.AfterTest(c)()
driver := localstore.Driver{Driver: goleveldb.MemoryDriver{}}
store, err := driver.Open("memory")
c.Assert(err, IsNil)
defer store.Close()
handle, err := infoschema.NewHandle(store)
c.Assert(err, IsNil)
dbName := model.NewCIStr("Test")
tbName := model.NewCIStr("T")
colName := model.NewCIStr("A")
idxName := model.NewCIStr("idx")
noexist := model.NewCIStr("noexist")
colID, err := genGlobalID(store)
c.Assert(err, IsNil)
colInfo := &model.ColumnInfo{
ID: colID,
Name: colName,
Offset: 0,
FieldType: *types.NewFieldType(mysql.TypeLonglong),
State: model.StatePublic,
}
idxInfo := &model.IndexInfo{
Name: idxName,
Table: tbName,
Columns: []*model.IndexColumn{
{
Name: colName,
Offset: 0,
Length: 10,
},
},
Unique: true,
Primary: true,
State: model.StatePublic,
}
tbID, err := genGlobalID(store)
c.Assert(err, IsNil)
tblInfo := &model.TableInfo{
ID: tbID,
Name: tbName,
Columns: []*model.ColumnInfo{colInfo},
Indices: []*model.IndexInfo{idxInfo},
State: model.StatePublic,
}
dbID, err := genGlobalID(store)
c.Assert(err, IsNil)
dbInfo := &model.DBInfo{
ID: dbID,
Name: dbName,
Tables: []*model.TableInfo{tblInfo},
State: model.StatePublic,
}
dbInfos := []*model.DBInfo{dbInfo}
err = kv.RunInNewTxn(store, true, func(txn kv.Transaction) error {
meta.NewMeta(txn).CreateDatabase(dbInfo)
return errors.Trace(err)
})
c.Assert(err, IsNil)
builder, err := infoschema.NewBuilder(handle).InitWithDBInfos(dbInfos, 1)
c.Assert(err, IsNil)
txn, err := store.Begin()
c.Assert(err, IsNil)
checkApplyCreateNonExistsSchemaDoesNotPanic(c, txn, builder)
checkApplyCreateNonExistsTableDoesNotPanic(c, txn, builder, dbID)
txn.Rollback()
err = builder.Build()
c.Assert(err, IsNil)
is := handle.Get()
schemaNames := is.AllSchemaNames()
c.Assert(schemaNames, HasLen, 3)
c.Assert(testutil.CompareUnorderedStringSlice(schemaNames, []string{infoschema.Name, perfschema.Name, "Test"}), IsTrue)
schemas := is.AllSchemas()
c.Assert(schemas, HasLen, 3)
schemas = is.Clone()
c.Assert(schemas, HasLen, 3)
c.Assert(is.SchemaExists(dbName), IsTrue)
c.Assert(is.SchemaExists(noexist), IsFalse)
schema, ok := is.SchemaByID(dbID)
c.Assert(ok, IsTrue)
c.Assert(schema, NotNil)
schema, ok = is.SchemaByID(tbID)
c.Assert(ok, IsFalse)
c.Assert(schema, IsNil)
schema, ok = is.SchemaByName(dbName)
c.Assert(ok, IsTrue)
c.Assert(schema, NotNil)
schema, ok = is.SchemaByName(noexist)
c.Assert(ok, IsFalse)
c.Assert(schema, IsNil)
c.Assert(is.TableExists(dbName, tbName), IsTrue)
c.Assert(is.TableExists(dbName, noexist), IsFalse)
tb, ok := is.TableByID(tbID)
c.Assert(ok, IsTrue)
c.Assert(tb, NotNil)
tb, ok = is.TableByID(dbID)
c.Assert(ok, IsFalse)
c.Assert(tb, IsNil)
alloc, ok := is.AllocByID(tbID)
c.Assert(ok, IsTrue)
c.Assert(alloc, NotNil)
tb, err = is.TableByName(dbName, tbName)
c.Assert(err, IsNil)
c.Assert(tb, NotNil)
tb, err = is.TableByName(dbName, noexist)
c.Assert(err, NotNil)
tbs := is.SchemaTables(dbName)
c.Assert(tbs, HasLen, 1)
tbs = is.SchemaTables(noexist)
c.Assert(tbs, HasLen, 0)
// Make sure partitions table exists
tb, err = is.TableByName(model.NewCIStr("information_schema"), model.NewCIStr("partitions"))
c.Assert(err, IsNil)
c.Assert(tb, NotNil)
}