// CopySchemaShard copies the schema from a source tablet to the
// specified shard. The schema is applied directly on the master of
// the destination shard, and is propogated to the replicas through
// binlogs.
func (wr *Wrangler) CopySchemaShard(ctx context.Context, sourceTabletAlias *pb.TabletAlias, tables, excludeTables []string, includeViews bool, destKeyspace, destShard string) error {
destShardInfo, err := wr.ts.GetShard(ctx, destKeyspace, destShard)
if err != nil {
return err
}
sourceSd, err := wr.GetSchema(ctx, sourceTabletAlias, tables, excludeTables, includeViews)
if err != nil {
return err
}
destSd, err := wr.GetSchema(ctx, destShardInfo.MasterAlias, tables, excludeTables, includeViews)
if err != nil {
destSd = nil
}
if destSd != nil {
diffs := myproto.DiffSchemaToArray("source", sourceSd, "dest", destSd)
if diffs == nil {
// Return early because dest has already the same schema as source.
return nil
}
}
createSQL := sourceSd.ToSQLStrings()
destTabletInfo, err := wr.ts.GetTablet(ctx, destShardInfo.MasterAlias)
if err != nil {
return err
}
for i, sqlLine := range createSQL {
err = wr.applySQLShard(ctx, destTabletInfo, sqlLine, i == len(createSQL)-1)
if err != nil {
return err
}
}
return nil
}
func (wr *Wrangler) applySchemaShardSimple(ctx context.Context, statusArray []*tabletStatus, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, force bool) (*myproto.SchemaChangeResult, error) {
// check all tablets have the same schema as the master's
// BeforeSchema. If not, we shouldn't proceed
log.Infof("Checking schema on all tablets")
for _, status := range statusArray {
diffs := myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.Alias, strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.Alias, strings.Join(diffs, "\n"))
}
}
}
// we're good, just send to the master
log.Infof("Applying schema change to master in simple mode")
sc := &myproto.SchemaChange{Sql: change, Force: force, AllowReplication: true, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
return wr.ApplySchema(ctx, masterTabletAlias, sc)
}
func (exec *TabletExecutor) preflightSchemaChanges(ctx context.Context, sqls []string) error {
exec.schemaDiffs = make([]*proto.SchemaChangeResult, len(sqls))
for i := range sqls {
schemaDiff, err := exec.tmClient.PreflightSchema(
ctx, exec.tabletInfos[0], sqls[i])
if err != nil {
return err
}
exec.schemaDiffs[i] = schemaDiff
diffs := proto.DiffSchemaToArray(
"BeforeSchema",
exec.schemaDiffs[i].BeforeSchema,
"AfterSchema",
exec.schemaDiffs[i].AfterSchema)
if len(diffs) == 0 {
return fmt.Errorf("Schema change: '%s' does not introduce any table definition change.", sqls[i])
}
}
return nil
}
func (wr *Wrangler) applySchemaShardComplex(ctx context.Context, statusArray []*tabletStatus, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, newParentTabletAlias *pb.TabletAlias, force bool, waitSlaveTimeout time.Duration) (*myproto.SchemaChangeResult, error) {
// apply the schema change to all replica / slave tablets
for _, status := range statusArray {
// if already applied, we skip this guy
diffs := myproto.DiffSchemaToArray("after", preflight.AfterSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) == 0 {
log.Infof("Tablet %v already has the AfterSchema, skipping", status.ti.Alias)
continue
}
// make sure the before schema matches
diffs = myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topo.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.Alias, strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.Alias, strings.Join(diffs, "\n"))
}
}
// take this guy out of the serving graph if necessary
ti, err := wr.ts.GetTablet(ctx, status.ti.Alias)
if err != nil {
return nil, err
}
typeChangeRequired := ti.IsInServingGraph()
if typeChangeRequired {
// note we want to update the serving graph there
err = wr.changeTypeInternal(ctx, ti.Alias, pb.TabletType_SCHEMA_UPGRADE)
if err != nil {
return nil, err
}
}
// apply the schema change
log.Infof("Applying schema change to slave %v in complex mode", status.ti.Alias)
sc := &myproto.SchemaChange{Sql: change, Force: force, AllowReplication: false, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
_, err = wr.ApplySchema(ctx, status.ti.Alias, sc)
if err != nil {
return nil, err
}
// put this guy back into the serving graph
if typeChangeRequired {
err = wr.changeTypeInternal(ctx, ti.Alias, ti.Tablet.Type)
if err != nil {
return nil, err
}
}
}
// if newParentTabletAlias is passed in, use that as the new master
if !topo.TabletAliasIsZero(newParentTabletAlias) {
log.Infof("Reparenting with new master set to %v", newParentTabletAlias)
oldMasterAlias := shardInfo.MasterAlias
// Create reusable Reparent event with available info
ev := &events.Reparent{}
if err := wr.plannedReparentShardLocked(ctx, ev, shardInfo.Keyspace(), shardInfo.ShardName(), newParentTabletAlias, waitSlaveTimeout); err != nil {
return nil, err
}
// Here we would apply the schema change to the old
// master, but we just scrap it, to be consistent
// with the previous implementation of the reparent.
// (this code will be refactored at some point anyway)
if err := wr.Scrap(ctx, oldMasterAlias, false, false); err != nil {
wr.Logger().Warningf("Scrapping old master %v from shard %v/%v failed: %v", oldMasterAlias, shardInfo.Keyspace(), shardInfo.ShardName(), err)
}
}
return &myproto.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil
}
// ApplySchemaChange will apply the schema change to the given database.
func (mysqld *Mysqld) ApplySchemaChange(dbName string, change *proto.SchemaChange) (*proto.SchemaChangeResult, error) {
// check current schema matches
beforeSchema, err := mysqld.GetSchema(dbName, nil, nil, false)
if err != nil {
return nil, err
}
if change.BeforeSchema != nil {
schemaDiffs := proto.DiffSchemaToArray("actual", beforeSchema, "expected", change.BeforeSchema)
if len(schemaDiffs) > 0 {
for _, msg := range schemaDiffs {
log.Warningf("BeforeSchema differs: %v", msg)
}
// let's see if the schema was already applied
if change.AfterSchema != nil {
schemaDiffs = proto.DiffSchemaToArray("actual", beforeSchema, "expected", change.AfterSchema)
if len(schemaDiffs) == 0 {
// no diff between the schema we expect
// after the change and the current
// schema, we already applied it
return &proto.SchemaChangeResult{BeforeSchema: beforeSchema, AfterSchema: beforeSchema}, nil
}
}
if change.Force {
log.Warningf("BeforeSchema differs, applying anyway")
} else {
return nil, fmt.Errorf("BeforeSchema differs")
}
}
}
sql := change.Sql
if !change.AllowReplication {
sql = "SET sql_log_bin = 0;\n" + sql
}
// add a 'use XXX' in front of the SQL
sql = "USE " + dbName + ";\n" + sql
// execute the schema change using an external mysql process
// (to benefit from the extra commands in mysql cli)
if err = mysqld.ExecuteMysqlCommand(sql); err != nil {
return nil, err
}
// get AfterSchema
afterSchema, err := mysqld.GetSchema(dbName, nil, nil, false)
if err != nil {
return nil, err
}
// compare to the provided AfterSchema
if change.AfterSchema != nil {
schemaDiffs := proto.DiffSchemaToArray("actual", afterSchema, "expected", change.AfterSchema)
if len(schemaDiffs) > 0 {
for _, msg := range schemaDiffs {
log.Warningf("AfterSchema differs: %v", msg)
}
if change.Force {
log.Warningf("AfterSchema differs, not reporting error")
} else {
return nil, fmt.Errorf("AfterSchema differs")
}
}
}
return &proto.SchemaChangeResult{BeforeSchema: beforeSchema, AfterSchema: afterSchema}, nil
}
func (wr *Wrangler) applySchemaKeyspace(keyspace string, change string, simple, force bool) (*myproto.SchemaChangeResult, error) {
shards, err := wr.ts.GetShardNames(keyspace)
if err != nil {
return nil, err
}
// corner cases
if len(shards) == 0 {
return nil, fmt.Errorf("No shards in keyspace %v", keyspace)
}
if len(shards) == 1 {
log.Infof("Only one shard in keyspace %v, using ApplySchemaShard", keyspace)
return wr.ApplySchemaShard(keyspace, shards[0], change, topo.TabletAlias{}, simple, force)
}
// Get schema on all shard masters in parallel
log.Infof("Getting schema on all shards")
beforeSchemas := make([]*myproto.SchemaDefinition, len(shards))
shardInfos := make([]*topo.ShardInfo, len(shards))
wg := sync.WaitGroup{}
mu := sync.Mutex{}
getErrs := make([]string, 0, 5)
for i, shard := range shards {
wg.Add(1)
go func(i int, shard string) {
var err error
defer func() {
if err != nil {
mu.Lock()
getErrs = append(getErrs, err.Error())
mu.Unlock()
}
wg.Done()
}()
shardInfos[i], err = wr.ts.GetShard(keyspace, shard)
if err != nil {
return
}
beforeSchemas[i], err = wr.GetSchema(shardInfos[i].MasterAlias, nil, nil, false)
}(i, shard)
}
wg.Wait()
if len(getErrs) > 0 {
return nil, fmt.Errorf("Error(s) getting schema: %v", strings.Join(getErrs, ", "))
}
// check they all match, or use the force flag
log.Infof("Checking starting schemas match on all shards")
for i, beforeSchema := range beforeSchemas {
if i == 0 {
continue
}
diffs := myproto.DiffSchemaToArray("shard 0", beforeSchemas[0], fmt.Sprintf("shard %v", i), beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Shard %v has inconsistent schema, ignoring: %v", i, strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Shard %v has inconsistent schema: %v", i, strings.Join(diffs, "\n"))
}
}
}
// preflight on shard 0 master, to get baseline
// this assumes shard 0 master doesn't have the schema upgrade applied
// if it does, we'll have to fix the slaves and other shards manually.
log.Infof("Running Preflight on Shard 0 Master")
preflight, err := wr.PreflightSchema(shardInfos[0].MasterAlias, change)
if err != nil {
return nil, err
}
// for each shard, apply the change
log.Infof("Applying change on all shards")
var applyErr error
for i, shard := range shards {
wg.Add(1)
go func(i int, shard string) {
defer wg.Done()
_, err := wr.lockAndApplySchemaShard(shardInfos[i], preflight, keyspace, shard, shardInfos[i].MasterAlias, change, topo.TabletAlias{}, simple, force)
if err != nil {
mu.Lock()
applyErr = err
mu.Unlock()
return
}
}(i, shard)
}
wg.Wait()
if applyErr != nil {
return nil, applyErr
}
return &myproto.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil
}
func (wr *Wrangler) applySchemaShardComplex(statusArray []*TabletStatus, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias topo.TabletAlias, change string, newParentTabletAlias topo.TabletAlias, force bool) (*myproto.SchemaChangeResult, error) {
// apply the schema change to all replica / slave tablets
for _, status := range statusArray {
// if already applied, we skip this guy
diffs := myproto.DiffSchemaToArray("after", preflight.AfterSchema, status.ti.Alias.String(), status.beforeSchema)
if len(diffs) == 0 {
log.Infof("Tablet %v already has the AfterSchema, skipping", status.ti.Alias)
continue
}
// make sure the before schema matches
diffs = myproto.DiffSchemaToArray("master", preflight.BeforeSchema, status.ti.Alias.String(), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.Alias, strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.Alias, strings.Join(diffs, "\n"))
}
}
// take this guy out of the serving graph if necessary
ti, err := wr.ts.GetTablet(status.ti.Alias)
if err != nil {
return nil, err
}
typeChangeRequired := ti.Tablet.IsInServingGraph()
if typeChangeRequired {
// note we want to update the serving graph there
err = wr.changeTypeInternal(ti.Alias, topo.TYPE_SCHEMA_UPGRADE)
if err != nil {
return nil, err
}
}
// apply the schema change
log.Infof("Applying schema change to slave %v in complex mode", status.ti.Alias)
sc := &myproto.SchemaChange{Sql: change, Force: force, AllowReplication: false, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
_, err = wr.ApplySchema(status.ti.Alias, sc)
if err != nil {
return nil, err
}
// put this guy back into the serving graph
if typeChangeRequired {
err = wr.changeTypeInternal(ti.Alias, ti.Tablet.Type)
if err != nil {
return nil, err
}
}
}
// if newParentTabletAlias is passed in, use that as the new master
if !newParentTabletAlias.IsZero() {
log.Infof("Reparenting with new master set to %v", newParentTabletAlias)
tabletMap, err := topo.GetTabletMapForShard(wr.ts, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, err
}
slaveTabletMap, masterTabletMap := sortedTabletMap(tabletMap)
newMasterTablet, err := wr.ts.GetTablet(newParentTabletAlias)
if err != nil {
return nil, err
}
err = wr.reparentShardGraceful(shardInfo, slaveTabletMap, masterTabletMap, newMasterTablet /*leaveMasterReadOnly*/, false)
if err != nil {
return nil, err
}
// Here we would apply the schema change to the old
// master, but after a reparent it's in Scrap state,
// so no need to. When/if reparent leaves the
// original master in a different state (like replica
// or rdonly), then we should apply the schema there
// too.
log.Infof("Skipping schema change on old master %v in complex mode, it's been Scrapped", masterTabletAlias)
}
return &myproto.SchemaChangeResult{BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}, nil
}
func (wr *Wrangler) applySchemaShard(ctx context.Context, shardInfo *topo.ShardInfo, preflight *myproto.SchemaChangeResult, masterTabletAlias *pb.TabletAlias, change string, newParentTabletAlias *pb.TabletAlias, force bool, waitSlaveTimeout time.Duration) (*myproto.SchemaChangeResult, error) {
// find all the shards we need to handle
aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, shardInfo.Keyspace(), shardInfo.ShardName())
if err != nil {
return nil, err
}
// build the array of tabletStatus we're going to use
statusArray := make([]*tabletStatus, 0, len(aliases)-1)
for _, alias := range aliases {
if alias == masterTabletAlias {
// we skip the master
continue
}
ti, err := wr.ts.GetTablet(ctx, alias)
if err != nil {
return nil, err
}
statusArray = append(statusArray, &tabletStatus{ti: ti})
}
// get schema on all tablets.
log.Infof("Getting schema on all tablets for shard %v/%v", shardInfo.Keyspace(), shardInfo.ShardName())
wg := &sync.WaitGroup{}
for _, status := range statusArray {
wg.Add(1)
go func(status *tabletStatus) {
status.beforeSchema, status.lastError = wr.tmc.GetSchema(ctx, status.ti, nil, nil, false)
wg.Done()
}(status)
}
wg.Wait()
// quick check for errors
for _, status := range statusArray {
if status.lastError != nil {
return nil, fmt.Errorf("Error getting schema on tablet %v: %v", status.ti.AliasString(), status.lastError)
}
}
// check all tablets have the same schema as the master's
// BeforeSchema. If not, we shouldn't proceed
log.Infof("Checking schema on all tablets")
for _, status := range statusArray {
diffs := myproto.DiffSchemaToArray("master", preflight.BeforeSchema, topoproto.TabletAliasString(status.ti.Alias), status.beforeSchema)
if len(diffs) > 0 {
if force {
log.Warningf("Tablet %v has inconsistent schema, ignoring: %v", status.ti.AliasString(), strings.Join(diffs, "\n"))
} else {
return nil, fmt.Errorf("Tablet %v has inconsistent schema: %v", status.ti.AliasString(), strings.Join(diffs, "\n"))
}
}
}
// we're good, just send to the master
log.Infof("Applying schema change to master")
sc := &myproto.SchemaChange{Sql: change, Force: force, AllowReplication: true, BeforeSchema: preflight.BeforeSchema, AfterSchema: preflight.AfterSchema}
return wr.ApplySchema(ctx, masterTabletAlias, sc)
}