// NewTabletServer creates an instance of TabletServer. Only one instance
// of TabletServer can be created per process.
func NewTabletServer(config Config) *TabletServer {
tsv := &TabletServer{
config: config,
QueryTimeout: sync2.NewAtomicDuration(time.Duration(config.QueryTimeout * 1e9)),
BeginTimeout: sync2.NewAtomicDuration(time.Duration(config.TxPoolTimeout * 1e9)),
checkMySQLThrottler: sync2.NewSemaphore(1, 0),
streamHealthMap: make(map[int]chan<- *querypb.StreamHealthResponse),
sessionID: Rand(),
history: history.New(10),
}
tsv.qe = NewQueryEngine(tsv, config)
tsv.invalidator = NewRowcacheInvalidator(config.StatsPrefix, tsv, tsv.qe, config.EnablePublishStats)
if config.EnablePublishStats {
stats.Publish(config.StatsPrefix+"TabletState", stats.IntFunc(func() int64 {
tsv.mu.Lock()
state := tsv.state
tsv.mu.Unlock()
return state
}))
stats.Publish(config.StatsPrefix+"QueryTimeout", stats.DurationFunc(tsv.QueryTimeout.Get))
stats.Publish(config.StatsPrefix+"BeginTimeout", stats.DurationFunc(tsv.BeginTimeout.Get))
stats.Publish(config.StatsPrefix+"TabletStateName", stats.StringFunc(tsv.GetState))
}
return tsv
}
func NewQueryEngine(config Config) *QueryEngine {
qe := &QueryEngine{}
qe.cachePool = NewCachePool("CachePool", config.RowCache, time.Duration(config.QueryTimeout*1e9), time.Duration(config.IdleTimeout*1e9))
qe.schemaInfo = NewSchemaInfo(config.QueryCacheSize, time.Duration(config.SchemaReloadTime*1e9), time.Duration(config.IdleTimeout*1e9))
qe.connPool = NewConnectionPool("ConnPool", config.PoolSize, time.Duration(config.IdleTimeout*1e9))
qe.streamConnPool = NewConnectionPool("StreamConnPool", config.StreamPoolSize, time.Duration(config.IdleTimeout*1e9))
qe.streamTokens = sync2.NewSemaphore(config.StreamExecThrottle, time.Duration(config.StreamWaitTimeout*1e9))
qe.reservedPool = NewReservedPool("ReservedPool")
qe.txPool = NewConnectionPool("TxPool", config.TransactionCap, time.Duration(config.IdleTimeout*1e9)) // connections in pool has to be > transactionCap
qe.activeTxPool = NewActiveTxPool("ActiveTxPool", time.Duration(config.TransactionTimeout*1e9))
qe.activePool = NewActivePool("ActivePool", time.Duration(config.QueryTimeout*1e9), time.Duration(config.IdleTimeout*1e9))
qe.consolidator = NewConsolidator()
qe.spotCheckFreq = sync2.AtomicInt64(config.SpotCheckRatio * SPOT_CHECK_MULTIPLIER)
qe.maxResultSize = sync2.AtomicInt64(config.MaxResultSize)
qe.streamBufferSize = sync2.AtomicInt64(config.StreamBufferSize)
stats.Publish("MaxResultSize", stats.IntFunc(qe.maxResultSize.Get))
stats.Publish("StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get))
queryStats = stats.NewTimings("Queries")
stats.NewRates("QPS", queryStats, 15, 60e9)
waitStats = stats.NewTimings("Waits")
killStats = stats.NewCounters("Kills")
errorStats = stats.NewCounters("Errors")
resultStats = stats.NewHistogram("Results", resultBuckets)
stats.Publish("SpotCheckRatio", stats.FloatFunc(func() float64 {
return float64(qe.spotCheckFreq.Get()) / SPOT_CHECK_MULTIPLIER
}))
spotCheckCount = stats.NewInt("SpotCheckCount")
return qe
}
func init() {
// The zookeeper C module logs quite a bit of useful information,
// but much of it does not come back in the error API. To aid
// debugging, enable the log to stderr for warnings.
//zookeeper.SetLogLevel(zookeeper.LOG_WARN)
maxConcurrency := 64
x := os.Getenv("ZK_CLIENT_MAX_CONCURRENCY")
if x != "" {
var err error
maxConcurrency, err = strconv.Atoi(x)
if err != nil {
log.Infof("invalid ZK_CLIENT_MAX_CONCURRENCY: %v", err)
}
}
sem = sync2.NewSemaphore(maxConcurrency, 0)
}
// NewTabletServer creates an instance of TabletServer. Only one instance
// of TabletServer can be created per process.
func NewTabletServer(config Config) *TabletServer {
tsv := &TabletServer{
config: config,
checkMySQLThrottler: sync2.NewSemaphore(1, 0),
streamHealthMap: make(map[int]chan<- *pb.StreamHealthResponse),
sessionID: Rand(),
}
tsv.qe = NewQueryEngine(tsv, config)
tsv.invalidator = NewRowcacheInvalidator(config.StatsPrefix, tsv, tsv.qe, config.EnablePublishStats)
if config.EnablePublishStats {
stats.Publish(config.StatsPrefix+"TabletState", stats.IntFunc(func() int64 {
tsv.mu.Lock()
state := tsv.state
tsv.mu.Unlock()
return state
}))
stats.Publish(config.StatsPrefix+"TabletStateName", stats.StringFunc(tsv.GetState))
}
return tsv
}
// clone phase:
// - copy the data from source tablets to destination masters (with replication on)
// Assumes that the schema has already been created on each destination tablet
// (probably from vtctl's CopySchemaShard)
func (vscw *VerticalSplitCloneWorker) clone(ctx context.Context) error {
vscw.setState(WorkerStateCloneOffline)
start := time.Now()
defer func() {
statsStateDurationsNs.Set(string(WorkerStateCloneOffline), time.Now().Sub(start).Nanoseconds())
}()
// get source schema
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sourceSchemaDefinition, err := vscw.wr.GetSchema(shortCtx, vscw.sourceAlias, vscw.tables, nil, true)
cancel()
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", topoproto.TabletAliasString(vscw.sourceAlias), err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter")
}
vscw.wr.Logger().Infof("Source tablet has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
vscw.tableStatusList.initialize(sourceSchemaDefinition)
// In parallel, setup the channels to send SQL data chunks to
// for each destination tablet.
//
// mu protects firstError
mu := sync.Mutex{}
var firstError error
ctx, cancelCopy := context.WithCancel(ctx)
processError := func(format string, args ...interface{}) {
vscw.wr.Logger().Errorf(format, args...)
mu.Lock()
if firstError == nil {
firstError = fmt.Errorf(format, args...)
cancelCopy()
}
mu.Unlock()
}
destinationWaitGroup := sync.WaitGroup{}
// we create one channel for the destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannel := make(chan string, vscw.destinationWriterCount*2)
// Set up the throttler for the destination shard.
keyspaceAndShard := topoproto.KeyspaceShardString(vscw.destinationKeyspace, vscw.destinationShard)
destinationThrottler, err := throttler.NewThrottler(
keyspaceAndShard, "transactions", vscw.destinationWriterCount, vscw.maxTPS, throttler.ReplicationLagModuleDisabled)
if err != nil {
return fmt.Errorf("cannot instantiate throttler: %v", err)
}
for j := 0; j < vscw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func(threadID int) {
defer destinationWaitGroup.Done()
defer destinationThrottler.ThreadFinished(threadID)
executor := newExecutor(vscw.wr, vscw.tsc, destinationThrottler, vscw.destinationKeyspace, vscw.destinationShard, threadID)
if err := executor.fetchLoop(ctx, insertChannel); err != nil {
processError("executer.FetchLoop failed: %v", err)
}
}(j)
}
// Now for each table, read data chunks and send them to insertChannel
sourceWaitGroup := sync.WaitGroup{}
sema := sync2.NewSemaphore(vscw.sourceReaderCount, 0)
dbName := vscw.destinationDbNames[topoproto.KeyspaceShardString(vscw.destinationKeyspace, vscw.destinationShard)]
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == tmutils.TableView {
continue
}
chunks, err := generateChunks(ctx, vscw.wr, vscw.sourceTablet, td, vscw.minTableSizeForSplit, vscw.sourceReaderCount)
if err != nil {
return err
}
vscw.tableStatusList.setThreadCount(tableIndex, len(chunks)-1)
for _, c := range chunks {
sourceWaitGroup.Add(1)
go func(td *tabletmanagerdatapb.TableDefinition, tableIndex int, chunk chunk) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
vscw.tableStatusList.threadStarted(tableIndex)
// Start streaming from the source tablet.
rr, err := NewRestartableResultReader(ctx, vscw.wr.Logger(), vscw.wr.TopoServer(), vscw.sourceAlias, td, chunk)
if err != nil {
processError("NewRestartableResultReader failed: %v", err)
return
}
defer rr.Close()
// process the data
if err := vscw.processData(ctx, dbName, td, tableIndex, rr, insertChannel, vscw.destinationPackCount); err != nil {
processError("ResultReader failed: %v", err)
}
vscw.tableStatusList.threadDone(tableIndex)
}(td, tableIndex, c)
}
}
sourceWaitGroup.Wait()
close(insertChannel)
destinationWaitGroup.Wait()
// Stop Throttler.
destinationThrottler.Close()
if firstError != nil {
return firstError
}
// then create and populate the blp_checkpoint table
if vscw.strategy.skipPopulateBlpCheckpoint {
vscw.wr.Logger().Infof("Skipping populating the blp_checkpoint table")
} else {
// get the current position from the source
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
status, err := vscw.wr.TabletManagerClient().SlaveStatus(shortCtx, vscw.sourceTablet)
cancel()
if err != nil {
return err
}
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if vscw.strategy.dontStartBinlogPlayer {
flags = binlogplayer.BlpFlagDontStart
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, vscw.maxTPS, throttler.ReplicationLagModuleDisabled, time.Now().Unix(), flags))
vscw.wr.Logger().Infof("Making and populating blp_checkpoint table")
if err := runSQLCommands(ctx, vscw.wr, vscw.tsc, vscw.destinationKeyspace, vscw.destinationShard, dbName, queries); err != nil {
processError("blp_checkpoint queries failed: %v", err)
}
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
if vscw.strategy.skipSetSourceShards {
vscw.wr.Logger().Infof("Skipping setting SourceShard on destination shard.")
} else {
vscw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", vscw.destinationKeyspace, vscw.destinationShard)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := vscw.wr.SetSourceShards(shortCtx, vscw.destinationKeyspace, vscw.destinationShard, []*topodatapb.TabletAlias{vscw.sourceAlias}, vscw.tables)
cancel()
if err != nil {
return fmt.Errorf("Failed to set source shards: %v", err)
}
}
err = vscw.findRefreshTargets(ctx)
if err != nil {
return fmt.Errorf("failed before refreshing state on destination tablets: %v", err)
}
// And force a state refresh (re-read topo) on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for _, tabletAlias := range vscw.refreshAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Refreshing state on tablet %v", ti.AliasString())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := vscw.wr.TabletManagerClient().RefreshState(shortCtx, ti.Tablet)
cancel()
if err != nil {
processError("RefreshState failed on tablet %v: %v", ti.AliasString(), err)
}
}(vscw.refreshTablets[*tabletAlias])
}
destinationWaitGroup.Wait()
return firstError
}
// copy phase:
// - copy the data from source tablets to destination masters (with replication on)
// Assumes that the schema has already been created on each destination tablet
// (probably from vtctl's CopySchemaShard)
func (scw *SplitCloneWorker) copy(ctx context.Context) error {
scw.setState(WorkerStateCopy)
// get source schema from the first shard
// TODO(alainjobart): for now, we assume the schema is compatible
// on all source shards. Furthermore, we estimate the number of rows
// in each source shard for each table to be about the same
// (rowCount is used to estimate an ETA)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sourceSchemaDefinition, err := scw.wr.GetSchema(shortCtx, scw.sourceAliases[0], nil, scw.excludeTables, true)
cancel()
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", topoproto.TabletAliasString(scw.sourceAliases[0]), err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter in tablet %v", topoproto.TabletAliasString(scw.sourceAliases[0]))
}
scw.wr.Logger().Infof("Source tablet 0 has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
scw.Mu.Lock()
scw.tableStatus = make([]*tableStatus, len(sourceSchemaDefinition.TableDefinitions))
for i, td := range sourceSchemaDefinition.TableDefinitions {
scw.tableStatus[i] = &tableStatus{
name: td.Name,
rowCount: td.RowCount * uint64(len(scw.sourceAliases)),
}
}
scw.startTime = time.Now()
scw.Mu.Unlock()
// Find the column index for the sharding columns in all the databases, and count rows
columnIndexes := make([]int, len(sourceSchemaDefinition.TableDefinitions))
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TableBaseTable {
// find the column to split on
columnIndexes[tableIndex] = -1
for i, name := range td.Columns {
if name == scw.keyspaceInfo.ShardingColumnName {
columnIndexes[tableIndex] = i
break
}
}
if columnIndexes[tableIndex] == -1 {
return fmt.Errorf("table %v doesn't have a column named '%v'", td.Name, scw.keyspaceInfo.ShardingColumnName)
}
scw.tableStatus[tableIndex].mu.Lock()
scw.tableStatus[tableIndex].rowCount = td.RowCount
scw.tableStatus[tableIndex].mu.Unlock()
} else {
scw.tableStatus[tableIndex].mu.Lock()
scw.tableStatus[tableIndex].isView = true
scw.tableStatus[tableIndex].mu.Unlock()
}
}
// In parallel, setup the channels to send SQL data chunks to for each destination tablet:
//
// mu protects the context for cancelation, and firstError
mu := sync.Mutex{}
var firstError error
ctx, cancelCopy := context.WithCancel(ctx)
processError := func(format string, args ...interface{}) {
scw.wr.Logger().Errorf(format, args...)
mu.Lock()
if firstError == nil {
firstError = fmt.Errorf(format, args...)
cancelCopy()
}
mu.Unlock()
}
insertChannels := make([]chan string, len(scw.destinationShards))
destinationWaitGroup := sync.WaitGroup{}
for shardIndex, si := range scw.destinationShards {
// we create one channel per destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannels[shardIndex] = make(chan string, scw.destinationWriterCount*2)
go func(shardName string, insertChannel chan string) {
for j := 0; j < scw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func() {
defer destinationWaitGroup.Done()
if err := executeFetchLoop(ctx, scw.wr, scw, shardName, insertChannel); err != nil {
processError("executeFetchLoop failed: %v", err)
}
}()
}
}(si.ShardName(), insertChannels[shardIndex])
}
// Now for each table, read data chunks and send them to all
// insertChannels
sourceWaitGroup := sync.WaitGroup{}
for shardIndex := range scw.sourceShards {
sema := sync2.NewSemaphore(scw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TableView {
continue
}
rowSplitter := NewRowSplitter(scw.destinationShards, key.ProtoToKeyspaceIdType(scw.keyspaceInfo.ShardingColumnType), columnIndexes[tableIndex])
chunks, err := FindChunks(ctx, scw.wr, scw.sourceTablets[shardIndex], td, scw.minTableSizeForSplit, scw.sourceReaderCount)
if err != nil {
return err
}
scw.tableStatus[tableIndex].setThreadCount(len(chunks) - 1)
for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ {
sourceWaitGroup.Add(1)
go func(td *myproto.TableDefinition, tableIndex, chunkIndex int) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
scw.tableStatus[tableIndex].threadStarted()
// build the query, and start the streaming
selectSQL := buildSQLFromChunks(scw.wr, td, chunks, chunkIndex, scw.sourceAliases[shardIndex].String())
qrr, err := NewQueryResultReaderForTablet(ctx, scw.wr.TopoServer(), scw.sourceAliases[shardIndex], selectSQL)
if err != nil {
processError("NewQueryResultReaderForTablet failed: %v", err)
return
}
defer qrr.Close()
// process the data
if err := scw.processData(td, tableIndex, qrr, rowSplitter, insertChannels, scw.destinationPackCount, ctx.Done()); err != nil {
processError("processData failed: %v", err)
}
scw.tableStatus[tableIndex].threadDone()
}(td, tableIndex, chunkIndex)
}
}
}
sourceWaitGroup.Wait()
for shardIndex := range scw.destinationShards {
close(insertChannels[shardIndex])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
// then create and populate the blp_checkpoint table
if scw.strategy.PopulateBlpCheckpoint {
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if scw.strategy.DontStartBinlogPlayer {
flags = binlogplayer.BlpFlagDontStart
}
// get the current position from the sources
for shardIndex := range scw.sourceShards {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
status, err := scw.wr.TabletManagerClient().SlaveStatus(shortCtx, scw.sourceTablets[shardIndex])
cancel()
if err != nil {
return err
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, time.Now().Unix(), flags))
}
for _, si := range scw.destinationShards {
destinationWaitGroup.Add(1)
go func(shardName string) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Making and populating blp_checkpoint table")
if err := runSQLCommands(ctx, scw.wr, scw, shardName, queries); err != nil {
processError("blp_checkpoint queries failed: %v", err)
}
}(si.ShardName())
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
// TODO(alainjobart) this is a superset, some shards may not
// overlap, have to deal with this better (for N -> M splits
// where both N>1 and M>1)
if scw.strategy.SkipSetSourceShards {
scw.wr.Logger().Infof("Skipping setting SourceShard on destination shards.")
} else {
for _, si := range scw.destinationShards {
scw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", si.Keyspace(), si.ShardName())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := scw.wr.SetSourceShards(shortCtx, si.Keyspace(), si.ShardName(), scw.sourceAliases, nil)
cancel()
if err != nil {
return fmt.Errorf("failed to set source shards: %v", err)
}
}
}
err = scw.findReloadTargets(ctx)
if err != nil {
return fmt.Errorf("failed before reloading schema on destination tablets: %v", err)
}
// And force a schema reload on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for shardIndex := range scw.destinationShards {
for _, tabletAlias := range scw.reloadAliases[shardIndex] {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Reloading schema on tablet %v", ti.AliasString())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := scw.wr.TabletManagerClient().ReloadSchema(shortCtx, ti)
cancel()
if err != nil {
processError("ReloadSchema failed on tablet %v: %v", ti.AliasString(), err)
}
}(scw.reloadTablets[shardIndex][*tabletAlias])
}
}
destinationWaitGroup.Wait()
return firstError
}
// restoreFiles will copy all the files from the BackupStorage to the
// right place
func restoreFiles(cnf *Mycnf, bh backupstorage.BackupHandle, fes []FileEntry, restoreConcurrency int) error {
sema := sync2.NewSemaphore(restoreConcurrency, 0)
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for i, fe := range fes {
wg.Add(1)
go func(i int, fe FileEntry) {
defer wg.Done()
// wait until we are ready to go, skip if we already
// encountered an error
sema.Acquire()
defer sema.Release()
if rec.HasErrors() {
return
}
// open the source file for reading
name := fmt.Sprintf("%v", i)
source, err := bh.ReadFile(name)
if err != nil {
rec.RecordError(err)
return
}
defer source.Close()
// open the destination file for writing
dstFile, err := fe.open(cnf, false)
if err != nil {
rec.RecordError(err)
return
}
defer func() { rec.RecordError(dstFile.Close()) }()
// create a buffering output
dst := bufio.NewWriterSize(dstFile, 2*1024*1024)
// create hash to write the compressed data to
hasher := newHasher()
// create a Tee: we split the input into the hasher
// and into the gunziper
tee := io.TeeReader(source, hasher)
// create the uncompresser
gz, err := cgzip.NewReader(tee)
if err != nil {
rec.RecordError(err)
return
}
defer func() { rec.RecordError(gz.Close()) }()
// copy the data. Will also write to the hasher
if _, err = io.Copy(dst, gz); err != nil {
rec.RecordError(err)
return
}
// check the hash
hash := hasher.HashString()
if hash != fe.Hash {
rec.RecordError(fmt.Errorf("hash mismatch for %v, got %v expected %v", fe.Name, hash, fe.Hash))
return
}
// flush the buffer
rec.RecordError(dst.Flush())
}(i, fe)
}
wg.Wait()
return rec.Error()
}
func backupFiles(mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fes []FileEntry, replicationPosition replication.Position, backupConcurrency int) (err error) {
sema := sync2.NewSemaphore(backupConcurrency, 0)
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for i, fe := range fes {
wg.Add(1)
go func(i int, fe FileEntry) {
defer wg.Done()
// wait until we are ready to go, skip if we already
// encountered an error
sema.Acquire()
defer sema.Release()
if rec.HasErrors() {
return
}
// open the source file for reading
source, err := fe.open(mysqld.Cnf(), true)
if err != nil {
rec.RecordError(err)
return
}
defer source.Close()
// open the destination file for writing, and a buffer
name := fmt.Sprintf("%v", i)
wc, err := bh.AddFile(name)
if err != nil {
rec.RecordError(fmt.Errorf("cannot add file: %v", err))
return
}
defer func() { rec.RecordError(wc.Close()) }()
dst := bufio.NewWriterSize(wc, 2*1024*1024)
// create the hasher and the tee on top
hasher := newHasher()
tee := io.MultiWriter(dst, hasher)
// create the gzip compression filter
gzip, err := cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED)
if err != nil {
rec.RecordError(fmt.Errorf("cannot create gziper: %v", err))
return
}
// copy from the source file to gzip to tee to output file and hasher
_, err = io.Copy(gzip, source)
if err != nil {
rec.RecordError(fmt.Errorf("cannot copy data: %v", err))
return
}
// close gzip to flush it, after that the hash is good
if err = gzip.Close(); err != nil {
rec.RecordError(fmt.Errorf("cannot close gzip: %v", err))
return
}
// flush the buffer to finish writing, save the hash
rec.RecordError(dst.Flush())
fes[i].Hash = hasher.HashString()
}(i, fe)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// open the MANIFEST
wc, err := bh.AddFile(backupManifest)
if err != nil {
return fmt.Errorf("cannot add %v to backup: %v", backupManifest, err)
}
defer func() {
if closeErr := wc.Close(); err == nil {
err = closeErr
}
}()
// JSON-encode and write the MANIFEST
bm := &BackupManifest{
FileEntries: fes,
Position: replicationPosition,
}
data, err := json.MarshalIndent(bm, "", " ")
if err != nil {
return fmt.Errorf("cannot JSON encode %v: %v", backupManifest, err)
}
if _, err := wc.Write([]byte(data)); err != nil {
return fmt.Errorf("cannot write %v: %v", backupManifest, err)
}
return nil
}
func (vsdw *VerticalSplitDiffWorker) diff() error {
vsdw.setState(stateVSDDiff)
vsdw.wr.Logger().Infof("Gathering schema information...")
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
wg.Add(1)
go func() {
var err error
vsdw.destinationSchemaDefinition, err = vsdw.wr.GetSchema(vsdw.destinationAlias, nil, nil, false)
rec.RecordError(err)
vsdw.wr.Logger().Infof("Got schema from destination %v", vsdw.destinationAlias)
wg.Done()
}()
wg.Add(1)
go func() {
var err error
vsdw.sourceSchemaDefinition, err = vsdw.wr.GetSchema(vsdw.sourceAlias, nil, nil, false)
rec.RecordError(err)
vsdw.wr.Logger().Infof("Got schema from source %v", vsdw.sourceAlias)
wg.Done()
}()
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// Build a list of regexp to exclude tables from source schema
tableRegexps := make([]*regexp.Regexp, len(vsdw.shardInfo.SourceShards[0].Tables))
for i, table := range vsdw.shardInfo.SourceShards[0].Tables {
var err error
tableRegexps[i], err = regexp.Compile(table)
if err != nil {
return fmt.Errorf("cannot compile regexp %v for table: %v", table, err)
}
}
// Remove the tables we don't need from the source schema
newSourceTableDefinitions := make([]myproto.TableDefinition, 0, len(vsdw.destinationSchemaDefinition.TableDefinitions))
for _, tableDefinition := range vsdw.sourceSchemaDefinition.TableDefinitions {
found := false
for _, tableRegexp := range tableRegexps {
if tableRegexp.MatchString(tableDefinition.Name) {
found = true
break
}
}
if !found {
vsdw.wr.Logger().Infof("Removing table %v from source schema", tableDefinition.Name)
continue
}
newSourceTableDefinitions = append(newSourceTableDefinitions, tableDefinition)
}
vsdw.sourceSchemaDefinition.TableDefinitions = newSourceTableDefinitions
// Check the schema
vsdw.wr.Logger().Infof("Diffing the schema...")
rec = concurrency.AllErrorRecorder{}
myproto.DiffSchema("destination", vsdw.destinationSchemaDefinition, "source", vsdw.sourceSchemaDefinition, &rec)
if rec.HasErrors() {
vsdw.wr.Logger().Warningf("Different schemas: %v", rec.Error())
} else {
vsdw.wr.Logger().Infof("Schema match, good.")
}
// run the diffs, 8 at a time
vsdw.wr.Logger().Infof("Running the diffs...")
sem := sync2.NewSemaphore(8, 0)
for _, tableDefinition := range vsdw.destinationSchemaDefinition.TableDefinitions {
wg.Add(1)
go func(tableDefinition myproto.TableDefinition) {
defer wg.Done()
sem.Acquire()
defer sem.Release()
vsdw.wr.Logger().Infof("Starting the diff on table %v", tableDefinition.Name)
sourceQueryResultReader, err := TableScan(vsdw.wr.Logger(), vsdw.wr.TopoServer(), vsdw.sourceAlias, &tableDefinition)
if err != nil {
vsdw.wr.Logger().Errorf("TableScan(source) failed: %v", err)
return
}
defer sourceQueryResultReader.Close()
destinationQueryResultReader, err := TableScan(vsdw.wr.Logger(), vsdw.wr.TopoServer(), vsdw.destinationAlias, &tableDefinition)
if err != nil {
vsdw.wr.Logger().Errorf("TableScan(destination) failed: %v", err)
return
}
defer destinationQueryResultReader.Close()
differ, err := NewRowDiffer(sourceQueryResultReader, destinationQueryResultReader, &tableDefinition)
if err != nil {
vsdw.wr.Logger().Errorf("NewRowDiffer() failed: %v", err)
return
}
report, err := differ.Go(vsdw.wr.Logger())
if err != nil {
vsdw.wr.Logger().Errorf("Differ.Go failed: %v", err)
} else {
if report.HasDifferences() {
vsdw.wr.Logger().Errorf("Table %v has differences: %v", tableDefinition.Name, report.String())
} else {
vsdw.wr.Logger().Infof("Table %v checks out (%v rows processed, %v qps)", tableDefinition.Name, report.processedRows, report.processingQPS)
}
}
}(tableDefinition)
}
wg.Wait()
return nil
}
// copy phase:
// - get schema on the sources, filter tables
// - create tables on all destinations
// - copy the data
func (scw *SplitCloneWorker) copy() error {
scw.setState(stateSCCopy)
// get source schema from the first shard
// TODO(alainjobart): for now, we assume the schema is compatible
// on all source shards. Furthermore, we estimate the number of rows
// in each source shard for each table to be about the same
// (rowCount is used to estimate an ETA)
sourceSchemaDefinition, err := scw.wr.GetSchema(scw.sourceAliases[0], nil, scw.excludeTables, true)
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", scw.sourceAliases[0], err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter in tablet %v", scw.sourceAliases[0])
}
scw.wr.Logger().Infof("Source tablet 0 has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
scw.mu.Lock()
scw.tableStatus = make([]tableStatus, len(sourceSchemaDefinition.TableDefinitions))
for i, td := range sourceSchemaDefinition.TableDefinitions {
scw.tableStatus[i].name = td.Name
scw.tableStatus[i].rowCount = td.RowCount * uint64(len(scw.sourceAliases))
}
scw.startTime = time.Now()
scw.mu.Unlock()
// Create all the commands to create the destination schema:
// - createDbCmds will create the database and the tables
// - createViewCmds will create the views
// - alterTablesCmds will modify the tables at the end if needed
// (all need template substitution for {{.DatabaseName}})
createDbCmds := make([]string, 0, len(sourceSchemaDefinition.TableDefinitions)+1)
createDbCmds = append(createDbCmds, sourceSchemaDefinition.DatabaseSchema)
createViewCmds := make([]string, 0, 16)
alterTablesCmds := make([]string, 0, 16)
columnIndexes := make([]int, len(sourceSchemaDefinition.TableDefinitions))
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TABLE_BASE_TABLE {
// build the create and alter statements
create, alter, err := mysqlctl.MakeSplitCreateTableSql(scw.wr.Logger(), td.Schema, "{{.DatabaseName}}", td.Name, scw.strategy)
if err != nil {
return fmt.Errorf("MakeSplitCreateTableSql(%v) returned: %v", td.Name, err)
}
createDbCmds = append(createDbCmds, create)
if alter != "" {
alterTablesCmds = append(alterTablesCmds, alter)
}
// find the column to split on
columnIndexes[tableIndex] = -1
for i, name := range td.Columns {
if name == scw.keyspaceInfo.ShardingColumnName {
columnIndexes[tableIndex] = i
break
}
}
if columnIndexes[tableIndex] == -1 {
return fmt.Errorf("table %v doesn't have a column named '%v'", td.Name, scw.keyspaceInfo.ShardingColumnName)
}
scw.tableStatus[tableIndex].mu.Lock()
scw.tableStatus[tableIndex].state = "before table creation"
scw.tableStatus[tableIndex].rowCount = td.RowCount
scw.tableStatus[tableIndex].mu.Unlock()
} else {
scw.tableStatus[tableIndex].mu.Lock()
createViewCmds = append(createViewCmds, td.Schema)
scw.tableStatus[tableIndex].state = "before view creation"
scw.tableStatus[tableIndex].rowCount = 0
scw.tableStatus[tableIndex].mu.Unlock()
}
}
// For each destination tablet (in parallel):
// - create the schema
// - setup the channels to send SQL data chunks
//
// mu protects the abort channel for closing, and firstError
mu := sync.Mutex{}
abort := make(chan struct{})
var firstError error
processError := func(format string, args ...interface{}) {
scw.wr.Logger().Errorf(format, args...)
mu.Lock()
if abort != nil {
close(abort)
abort = nil
firstError = fmt.Errorf(format, args...)
}
mu.Unlock()
}
insertChannels := make([][]chan string, len(scw.destinationShards))
destinationWaitGroup := sync.WaitGroup{}
for shardIndex, _ := range scw.destinationShards {
insertChannels[shardIndex] = make([]chan string, len(scw.destinationAliases[shardIndex]))
for i, tabletAlias := range scw.destinationAliases[shardIndex] {
// we create one channel per destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannels[shardIndex][i] = make(chan string, scw.destinationWriterCount*2)
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo, insertChannel chan string) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Creating tables on tablet %v", ti.Alias)
if err := runSqlCommands(scw.wr, ti, createDbCmds, abort); err != nil {
processError("createDbCmds failed: %v", err)
return
}
if len(createViewCmds) > 0 {
scw.wr.Logger().Infof("Creating views on tablet %v", ti.Alias)
if err := runSqlCommands(scw.wr, ti, createViewCmds, abort); err != nil {
processError("createViewCmds failed: %v", err)
return
}
}
for j := 0; j < scw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func() {
defer destinationWaitGroup.Done()
if err := executeFetchLoop(scw.wr, ti, insertChannel, abort); err != nil {
processError("executeFetchLoop failed: %v", err)
}
}()
}
}(scw.destinationTablets[shardIndex][tabletAlias], insertChannels[shardIndex][i])
}
}
// Now for each table, read data chunks and send them to all
// insertChannels
sourceWaitGroup := sync.WaitGroup{}
for shardIndex, _ := range scw.sourceShards {
sema := sync2.NewSemaphore(scw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TABLE_VIEW {
continue
}
rowSplitter := NewRowSplitter(scw.destinationShards, scw.keyspaceInfo.ShardingColumnType, columnIndexes[tableIndex])
chunks, err := findChunks(scw.wr, scw.sourceTablets[shardIndex], td, scw.minTableSizeForSplit, scw.sourceReaderCount)
if err != nil {
return err
}
for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ {
sourceWaitGroup.Add(1)
go func(td *myproto.TableDefinition, tableIndex, chunkIndex int) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
// build the query, and start the streaming
selectSQL := buildSQLFromChunks(scw.wr, td, chunks, chunkIndex, scw.sourceAliases[shardIndex].String())
qrr, err := NewQueryResultReaderForTablet(scw.wr.TopoServer(), scw.sourceAliases[shardIndex], selectSQL)
if err != nil {
processError("NewQueryResultReaderForTablet failed: %v", err)
return
}
// process the data
if err := scw.processData(td, tableIndex, qrr, rowSplitter, insertChannels, abort); err != nil {
processError("processData failed: %v", err)
}
}(td, tableIndex, chunkIndex)
}
}
}
sourceWaitGroup.Wait()
for shardIndex, _ := range scw.destinationShards {
for _, c := range insertChannels[shardIndex] {
close(c)
}
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
// do the post-copy alters if any
if len(alterTablesCmds) > 0 {
for shardIndex, _ := range scw.destinationShards {
for _, tabletAlias := range scw.destinationAliases[shardIndex] {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Altering tables on tablet %v", ti.Alias)
if err := runSqlCommands(scw.wr, ti, alterTablesCmds, abort); err != nil {
processError("alterTablesCmds failed on tablet %v: %v", ti.Alias, err)
}
}(scw.destinationTablets[shardIndex][tabletAlias])
}
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// then create and populate the blp_checkpoint table
if strings.Index(scw.strategy, "populateBlpCheckpoint") != -1 {
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if strings.Index(scw.strategy, "dontStartBinlogPlayer") != -1 {
flags = binlogplayer.BLP_FLAG_DONT_START
}
// get the current position from the sources
for shardIndex, _ := range scw.sourceShards {
status, err := scw.wr.TabletManagerClient().SlaveStatus(scw.sourceTablets[shardIndex], 30*time.Second)
if err != nil {
return err
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, time.Now().Unix(), flags))
}
for shardIndex, _ := range scw.destinationShards {
for _, tabletAlias := range scw.destinationAliases[shardIndex] {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Making and populating blp_checkpoint table on tablet %v", ti.Alias)
if err := runSqlCommands(scw.wr, ti, queries, abort); err != nil {
processError("blp_checkpoint queries failed on tablet %v: %v", ti.Alias, err)
}
}(scw.destinationTablets[shardIndex][tabletAlias])
}
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
// TODO(alainjobart) this is a superset, some shards may not
// overlap, have to deal with this better (for N -> M splits
// where both N>1 and M>1)
if strings.Index(scw.strategy, "skipSetSourceShards") != -1 {
scw.wr.Logger().Infof("Skipping setting SourceShard on destination shards.")
} else {
for _, si := range scw.destinationShards {
scw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", si.Keyspace(), si.ShardName())
if err := scw.wr.SetSourceShards(si.Keyspace(), si.ShardName(), scw.sourceAliases, nil); err != nil {
return fmt.Errorf("Failed to set source shards: %v", err)
}
}
}
// And force a schema reload on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for shardIndex, _ := range scw.destinationShards {
for _, tabletAlias := range scw.destinationAliases[shardIndex] {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Reloading schema on tablet %v", ti.Alias)
if err := scw.wr.TabletManagerClient().ReloadSchema(ti, 30*time.Second); err != nil {
processError("ReloadSchema failed on tablet %v: %v", ti.Alias, err)
}
}(scw.destinationTablets[shardIndex][tabletAlias])
}
}
destinationWaitGroup.Wait()
return firstError
}
// MultiRestore is the main entry point for multi restore.
// - If the strategy contains the string 'writeBinLogs' then we will
// also write to the binary logs.
// - If the strategy contains the command 'populateBlpCheckpoint' then we
// will populate the blp_checkpoint table with master positions to start from
func (mysqld *Mysqld) MultiRestore(destinationDbName string, keyRange key.KeyRange, sourceAddrs []*url.URL, snapshotConcurrency, fetchConcurrency, insertTableConcurrency, fetchRetryCount int, strategy string) (err error) {
writeBinLogs := strings.Contains(strategy, "writeBinLogs")
manifests := make([]*SplitSnapshotManifest, len(sourceAddrs))
rc := concurrency.NewResourceConstraint(fetchConcurrency)
for i, sourceAddr := range sourceAddrs {
rc.Add(1)
go func(sourceAddr *url.URL, i int) {
rc.Acquire()
defer rc.ReleaseAndDone()
if rc.HasErrors() {
return
}
var sourceDbName string
if len(sourceAddr.Path) < 2 { // "" or "/"
sourceDbName = destinationDbName
} else {
sourceDbName = sourceAddr.Path[1:]
}
ssm, e := fetchSnapshotManifestWithRetry("http://"+sourceAddr.Host, sourceDbName, keyRange, fetchRetryCount)
manifests[i] = ssm
rc.RecordError(e)
}(sourceAddr, i)
}
if err = rc.Wait(); err != nil {
return
}
if e := SanityCheckManifests(manifests); e != nil {
return e
}
tempStoragePath := path.Join(mysqld.SnapshotDir, "multirestore", destinationDbName)
// Start fresh
if err = os.RemoveAll(tempStoragePath); err != nil {
return
}
if err = os.MkdirAll(tempStoragePath, 0775); err != nil {
return err
}
defer func() {
if e := os.RemoveAll(tempStoragePath); e != nil {
log.Errorf("error removing %v: %v", tempStoragePath, e)
}
}()
// Handle our concurrency:
// - fetchConcurrency tasks for network
// - insertTableConcurrency for table inserts from a file
// into an innodb table
// - snapshotConcurrency tasks for table inserts / modify tables
sems := make(map[string]*sync2.Semaphore, len(manifests[0].SchemaDefinition.TableDefinitions)+3)
sems["net"] = sync2.NewSemaphore(fetchConcurrency, 0)
sems["db"] = sync2.NewSemaphore(snapshotConcurrency, 0)
// Store the alter table statements for after restore,
// and how many jobs we're running on each table
// TODO(alainjobart) the jobCount map is a bit weird. replace it
// with a map of WaitGroups, initialized to the number of files
// per table. Have extra go routines for the tables with auto_increment
// to wait on the waitgroup, and apply the modify_table.
postSql := make(map[string]string, len(manifests[0].SchemaDefinition.TableDefinitions))
jobCount := make(map[string]*sync2.AtomicInt32)
// Create the database (it's a good check to know if we're running
// multirestore a second time too!)
manifest := manifests[0] // I am assuming they all match
createDatabase, e := fillStringTemplate(manifest.SchemaDefinition.DatabaseSchema, map[string]string{"DatabaseName": destinationDbName})
if e != nil {
return e
}
if createDatabase == "" {
return fmt.Errorf("Empty create database statement")
}
createDbCmds := make([]string, 0, len(manifest.SchemaDefinition.TableDefinitions)+2)
if !writeBinLogs {
createDbCmds = append(createDbCmds, "SET sql_log_bin = OFF")
}
createDbCmds = append(createDbCmds, createDatabase)
createDbCmds = append(createDbCmds, "USE `"+destinationDbName+"`")
createViewCmds := make([]string, 0, 16)
for _, td := range manifest.SchemaDefinition.TableDefinitions {
if td.Type == TABLE_BASE_TABLE {
createDbCmd, alterTable, err := makeCreateTableSql(td.Schema, td.Name, strategy)
if err != nil {
return err
}
if alterTable != "" {
postSql[td.Name] = alterTable
}
jobCount[td.Name] = new(sync2.AtomicInt32)
createDbCmds = append(createDbCmds, createDbCmd)
sems["table-"+td.Name] = sync2.NewSemaphore(insertTableConcurrency, 0)
} else {
// views are just created with the right db name
// and no data will ever go in them. We create them
// after all tables are created, as they will
// probably depend on real tables.
createViewCmd, err := fillStringTemplate(td.Schema, map[string]string{"DatabaseName": destinationDbName})
if err != nil {
return err
}
createViewCmds = append(createViewCmds, createViewCmd)
}
}
createDbCmds = append(createDbCmds, createViewCmds...)
if err = mysqld.executeSuperQueryList(createDbCmds); err != nil {
return
}
// compute how many jobs we will have
for _, manifest := range manifests {
for _, file := range manifest.Source.Files {
jobCount[file.TableName].Add(1)
}
}
loadDataInfile := `LOAD DATA INFILE '{{.TableInputPath}}' INTO TABLE {{.TableName}} CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n' ({{.Columns}})`
// fetch all the csv files, and apply them one at a time. Note
// this might start many go routines, and they'll all be
// waiting on the resource semaphores.
mrc := concurrency.NewMultiResourceConstraint(sems)
for manifestIndex, manifest := range manifests {
if err = os.Mkdir(path.Join(tempStoragePath, manifest.Source.Addr), 0775); err != nil {
return err
}
for i := range manifest.Source.Files {
lsf := localSnapshotFile{manifest: manifest, file: &manifest.Source.Files[i], basePath: tempStoragePath}
mrc.Add(1)
go func(manifestIndex, i int) {
defer mrc.Done()
// compute a few things now, so if we can't we
// don't take resources:
// - get the schema
td, ok := manifest.SchemaDefinition.GetTable(lsf.tableName())
if !ok {
mrc.RecordError(fmt.Errorf("No table named %v in schema", lsf.tableName()))
return
}
// - get the load data statement
queryParams := map[string]string{
"TableInputPath": lsf.filename(),
"TableName": lsf.tableName(),
"Columns": strings.Join(td.Columns, ", "),
}
loadStatement, e := fillStringTemplate(loadDataInfile, queryParams)
if e != nil {
mrc.RecordError(e)
return
}
// get the file, using the 'net' resource
mrc.Acquire("net")
if mrc.HasErrors() {
mrc.Release("net")
return
}
e = fetchFileWithRetry(lsf.url(), lsf.file.Hash, lsf.filename(), fetchRetryCount)
mrc.Release("net")
if e != nil {
mrc.RecordError(e)
return
}
defer os.Remove(lsf.filename())
// acquire the table lock (we do this first
// so we maximize access to db. Otherwise
// if 8 threads had gotten the db lock but
// were writing to the same table, only one
// load would go at once)
tableLockName := "table-" + lsf.tableName()
mrc.Acquire(tableLockName)
defer func() {
mrc.Release(tableLockName)
}()
if mrc.HasErrors() {
return
}
// acquire the db lock
mrc.Acquire("db")
defer func() {
mrc.Release("db")
}()
if mrc.HasErrors() {
return
}
// load the data in
queries := buildQueryList(destinationDbName, loadStatement, writeBinLogs)
e = mysqld.executeSuperQueryList(queries)
if e != nil {
mrc.RecordError(e)
return
}
// if we're running the last insert,
// potentially re-add the auto-increments
remainingInserts := jobCount[lsf.tableName()].Add(-1)
if remainingInserts == 0 && postSql[lsf.tableName()] != "" {
queries = buildQueryList(destinationDbName, postSql[lsf.tableName()], writeBinLogs)
e = mysqld.executeSuperQueryList(queries)
if e != nil {
mrc.RecordError(e)
return
}
}
}(manifestIndex, i)
}
}
if err = mrc.Wait(); err != nil {
return err
}
// populate blp_checkpoint table if we want to
if strings.Index(strategy, "populateBlpCheckpoint") != -1 {
queries := make([]string, 0, 4)
queries = append(queries, "USE `_vt`")
if !writeBinLogs {
queries = append(queries, "SET sql_log_bin = OFF")
queries = append(queries, "SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED")
}
for manifestIndex, manifest := range manifests {
insertRecovery := fmt.Sprintf(INSERT_INTO_RECOVERY,
manifestIndex,
manifest.Source.Addr,
manifest.Source.MasterState.ReplicationPosition.MasterLogFile,
manifest.Source.MasterState.ReplicationPosition.MasterLogPosition,
manifest.Source.MasterState.ReplicationPosition.MasterLogGroupId,
time.Now().Unix())
queries = append(queries, insertRecovery)
}
if err = mysqld.executeSuperQueryList(queries); err != nil {
return err
}
}
return nil
}
// copy phase:
// - get schema on the source, filter tables
// - create tables on all destinations
// - copy the data
func (vscw *VerticalSplitCloneWorker) copy() error {
vscw.setState(stateVSCCopy)
// get source schema
sourceSchemaDefinition, err := vscw.wr.GetSchema(vscw.sourceAlias, vscw.tables, nil, true)
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", vscw.sourceAlias, err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter")
}
vscw.wr.Logger().Infof("Source tablet has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
vscw.mu.Lock()
vscw.tableStatus = make([]tableStatus, len(sourceSchemaDefinition.TableDefinitions))
for i, td := range sourceSchemaDefinition.TableDefinitions {
vscw.tableStatus[i].name = td.Name
vscw.tableStatus[i].rowCount = td.RowCount
}
vscw.startTime = time.Now()
vscw.mu.Unlock()
// Create all the commands to create the destination schema:
// - createDbCmds will create the database and the tables
// - createViewCmds will create the views
// - alterTablesCmds will modify the tables at the end if needed
// (all need template substitution for {{.DatabaseName}})
createDbCmds := make([]string, 0, len(sourceSchemaDefinition.TableDefinitions)+1)
createDbCmds = append(createDbCmds, sourceSchemaDefinition.DatabaseSchema)
createViewCmds := make([]string, 0, 16)
alterTablesCmds := make([]string, 0, 16)
for i, td := range sourceSchemaDefinition.TableDefinitions {
vscw.tableStatus[i].mu.Lock()
if td.Type == myproto.TABLE_BASE_TABLE {
create, alter, err := mysqlctl.MakeSplitCreateTableSql(vscw.wr.Logger(), td.Schema, "{{.DatabaseName}}", td.Name, vscw.strategy)
if err != nil {
return fmt.Errorf("MakeSplitCreateTableSql(%v) returned: %v", td.Name, err)
}
createDbCmds = append(createDbCmds, create)
if alter != "" {
alterTablesCmds = append(alterTablesCmds, alter)
}
vscw.tableStatus[i].state = "before table creation"
vscw.tableStatus[i].rowCount = td.RowCount
} else {
createViewCmds = append(createViewCmds, td.Schema)
vscw.tableStatus[i].state = "before view creation"
vscw.tableStatus[i].rowCount = 0
}
vscw.tableStatus[i].mu.Unlock()
}
// For each destination tablet (in parallel):
// - create the schema
// - setup the channels to send SQL data chunks
//
// mu protects the abort channel for closing, and firstError
mu := sync.Mutex{}
abort := make(chan struct{})
var firstError error
processError := func(format string, args ...interface{}) {
vscw.wr.Logger().Errorf(format, args...)
mu.Lock()
if abort != nil {
close(abort)
abort = nil
firstError = fmt.Errorf(format, args...)
}
mu.Unlock()
}
insertChannels := make([]chan string, len(vscw.destinationAliases))
destinationWaitGroup := sync.WaitGroup{}
for i, tabletAlias := range vscw.destinationAliases {
// we create one channel per destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannels[i] = make(chan string, vscw.destinationWriterCount*2)
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo, insertChannel chan string) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Creating tables on tablet %v", ti.Alias)
if err := runSqlCommands(vscw.wr, ti, createDbCmds, abort); err != nil {
processError("createDbCmds failed: %v", err)
return
}
if len(createViewCmds) > 0 {
vscw.wr.Logger().Infof("Creating views on tablet %v", ti.Alias)
if err := runSqlCommands(vscw.wr, ti, createViewCmds, abort); err != nil {
processError("createViewCmds failed: %v", err)
return
}
}
for j := 0; j < vscw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func() {
defer destinationWaitGroup.Done()
if err := executeFetchLoop(vscw.wr, ti, insertChannel, abort); err != nil {
processError("executeFetchLoop failed: %v", err)
}
}()
}
}(vscw.destinationTablets[tabletAlias], insertChannels[i])
}
// Now for each table, read data chunks and send them to all
// insertChannels
sourceWaitGroup := sync.WaitGroup{}
sema := sync2.NewSemaphore(vscw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TABLE_VIEW {
vscw.tableStatus[tableIndex].setState("view created")
continue
}
vscw.tableStatus[tableIndex].setState("before copy")
chunks, err := findChunks(vscw.wr, vscw.sourceTablet, td, vscw.minTableSizeForSplit, vscw.sourceReaderCount)
if err != nil {
return err
}
for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ {
sourceWaitGroup.Add(1)
go func(td *myproto.TableDefinition, tableIndex, chunkIndex int) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
vscw.tableStatus[tableIndex].setState("started the copy")
// build the query, and start the streaming
selectSQL := buildSQLFromChunks(vscw.wr, td, chunks, chunkIndex, vscw.sourceAlias.String())
qrr, err := NewQueryResultReaderForTablet(vscw.wr.TopoServer(), vscw.sourceAlias, selectSQL)
if err != nil {
processError("NewQueryResultReaderForTablet failed: %v", err)
return
}
// process the data
if err := vscw.processData(td, tableIndex, qrr, insertChannels, abort); err != nil {
processError("QueryResultReader failed: %v", err)
}
}(td, tableIndex, chunkIndex)
}
}
sourceWaitGroup.Wait()
for _, c := range insertChannels {
close(c)
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
// do the post-copy alters if any
if len(alterTablesCmds) > 0 {
for _, tabletAlias := range vscw.destinationAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Altering tables on tablet %v", ti.Alias)
if err := runSqlCommands(vscw.wr, ti, alterTablesCmds, abort); err != nil {
processError("alterTablesCmds failed on tablet %v: %v", ti.Alias, err)
}
}(vscw.destinationTablets[tabletAlias])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// then create and populate the blp_checkpoint table
if strings.Index(vscw.strategy, "populateBlpCheckpoint") != -1 {
// get the current position from the source
status, err := vscw.wr.TabletManagerClient().SlaveStatus(vscw.sourceTablet, 30*time.Second)
if err != nil {
return err
}
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if strings.Index(vscw.strategy, "dontStartBinlogPlayer") != -1 {
flags = binlogplayer.BLP_FLAG_DONT_START
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, time.Now().Unix(), flags))
for _, tabletAlias := range vscw.destinationAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Making and populating blp_checkpoint table on tablet %v", ti.Alias)
if err := runSqlCommands(vscw.wr, ti, queries, abort); err != nil {
processError("blp_checkpoint queries failed on tablet %v: %v", ti.Alias, err)
}
}(vscw.destinationTablets[tabletAlias])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
if strings.Index(vscw.strategy, "skipSetSourceShards") != -1 {
vscw.wr.Logger().Infof("Skipping setting SourceShard on destination shard.")
} else {
vscw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", vscw.destinationKeyspace, vscw.destinationShard)
if err := vscw.wr.SetSourceShards(vscw.destinationKeyspace, vscw.destinationShard, []topo.TabletAlias{vscw.sourceAlias}, vscw.tables); err != nil {
return fmt.Errorf("Failed to set source shards: %v", err)
}
}
// And force a schema reload on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for _, tabletAlias := range vscw.destinationAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Reloading schema on tablet %v", ti.Alias)
if err := vscw.wr.TabletManagerClient().ReloadSchema(ti, 30*time.Second); err != nil {
processError("ReloadSchema failed on tablet %v: %v", ti.Alias, err)
}
}(vscw.destinationTablets[tabletAlias])
}
destinationWaitGroup.Wait()
return firstError
}
// copy phase:
// - copy the data from source tablets to destination masters (with replication on)
// Assumes that the schema has already been created on each destination tablet
// (probably from vtctl's CopySchemaShard)
func (scw *LegacySplitCloneWorker) copy(ctx context.Context) error {
scw.setState(WorkerStateCloneOffline)
start := time.Now()
defer func() {
statsStateDurationsNs.Set(string(WorkerStateCloneOffline), time.Now().Sub(start).Nanoseconds())
}()
// get source schema from the first shard
// TODO(alainjobart): for now, we assume the schema is compatible
// on all source shards. Furthermore, we estimate the number of rows
// in each source shard for each table to be about the same
// (rowCount is used to estimate an ETA)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sourceSchemaDefinition, err := scw.wr.GetSchema(shortCtx, scw.sourceAliases[0], nil, scw.excludeTables, false /* includeViews */)
cancel()
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", topoproto.TabletAliasString(scw.sourceAliases[0]), err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter in tablet %v", topoproto.TabletAliasString(scw.sourceAliases[0]))
}
for _, td := range sourceSchemaDefinition.TableDefinitions {
if len(td.Columns) == 0 {
return fmt.Errorf("schema for table %v has no columns", td.Name)
}
}
scw.wr.Logger().Infof("Source tablet 0 has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
scw.tableStatusList.initialize(sourceSchemaDefinition)
// In parallel, setup the channels to send SQL data chunks to for each destination tablet:
//
// mu protects the context for cancelation, and firstError
mu := sync.Mutex{}
var firstError error
ctx, cancelCopy := context.WithCancel(ctx)
processError := func(format string, args ...interface{}) {
scw.wr.Logger().Errorf(format, args...)
mu.Lock()
if firstError == nil {
firstError = fmt.Errorf(format, args...)
cancelCopy()
}
mu.Unlock()
}
insertChannels := make([]chan string, len(scw.destinationShards))
destinationWaitGroup := sync.WaitGroup{}
for shardIndex, si := range scw.destinationShards {
// we create one channel per destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannels[shardIndex] = make(chan string, scw.destinationWriterCount*2)
go func(keyspace, shard string, insertChannel chan string) {
for j := 0; j < scw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func(threadID int) {
defer destinationWaitGroup.Done()
keyspaceAndShard := topoproto.KeyspaceShardString(keyspace, shard)
throttler := scw.destinationThrottlers[keyspaceAndShard]
defer throttler.ThreadFinished(threadID)
executor := newExecutor(scw.wr, scw.tsc, throttler, keyspace, shard, threadID)
if err := executor.fetchLoop(ctx, insertChannel); err != nil {
processError("executer.FetchLoop failed: %v", err)
}
}(j)
}
}(si.Keyspace(), si.ShardName(), insertChannels[shardIndex])
}
// read the vschema if needed
var keyspaceSchema *vindexes.KeyspaceSchema
if *useV3ReshardingMode {
kschema, err := scw.wr.TopoServer().GetVSchema(ctx, scw.keyspace)
if err != nil {
return fmt.Errorf("cannot load VSchema for keyspace %v: %v", scw.keyspace, err)
}
if kschema == nil {
return fmt.Errorf("no VSchema for keyspace %v", scw.keyspace)
}
keyspaceSchema, err = vindexes.BuildKeyspaceSchema(kschema, scw.keyspace)
if err != nil {
return fmt.Errorf("cannot build vschema for keyspace %v: %v", scw.keyspace, err)
}
}
// Now for each table, read data chunks and send them to all
// insertChannels
sourceWaitGroup := sync.WaitGroup{}
for shardIndex := range scw.sourceShards {
sema := sync2.NewSemaphore(scw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
var keyResolver keyspaceIDResolver
if *useV3ReshardingMode {
keyResolver, err = newV3ResolverFromTableDefinition(keyspaceSchema, td)
if err != nil {
return fmt.Errorf("cannot resolve v3 sharding keys for keyspace %v: %v", scw.keyspace, err)
}
} else {
keyResolver, err = newV2Resolver(scw.keyspaceInfo, td)
if err != nil {
return fmt.Errorf("cannot resolve sharding keys for keyspace %v: %v", scw.keyspace, err)
}
}
rowSplitter := NewRowSplitter(scw.destinationShards, keyResolver)
chunks, err := generateChunks(ctx, scw.wr, scw.sourceTablets[shardIndex], td, scw.sourceReaderCount, defaultMinRowsPerChunk)
if err != nil {
return err
}
scw.tableStatusList.setThreadCount(tableIndex, len(chunks)-1)
for _, c := range chunks {
sourceWaitGroup.Add(1)
go func(td *tabletmanagerdatapb.TableDefinition, tableIndex int, chunk chunk) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
scw.tableStatusList.threadStarted(tableIndex)
// Start streaming from the source tablets.
tp := newSingleTabletProvider(ctx, scw.wr.TopoServer(), scw.sourceAliases[shardIndex])
rr, err := NewRestartableResultReader(ctx, scw.wr.Logger(), tp, td, chunk, false /* allowMultipleRetries */)
if err != nil {
processError("NewRestartableResultReader failed: %v", err)
return
}
defer rr.Close(ctx)
// process the data
dbNames := make([]string, len(scw.destinationShards))
for i, si := range scw.destinationShards {
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
dbNames[i] = scw.destinationDbNames[keyspaceAndShard]
}
if err := scw.processData(ctx, dbNames, td, tableIndex, rr, rowSplitter, insertChannels, scw.destinationPackCount); err != nil {
processError("processData failed: %v", err)
}
scw.tableStatusList.threadDone(tableIndex)
}(td, tableIndex, c)
}
}
}
sourceWaitGroup.Wait()
for shardIndex := range scw.destinationShards {
close(insertChannels[shardIndex])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
// then create and populate the blp_checkpoint table
if scw.strategy.skipPopulateBlpCheckpoint {
scw.wr.Logger().Infof("Skipping populating the blp_checkpoint table")
} else {
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if scw.strategy.dontStartBinlogPlayer {
flags = binlogplayer.BlpFlagDontStart
}
// get the current position from the sources
for shardIndex := range scw.sourceShards {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
status, err := scw.wr.TabletManagerClient().SlaveStatus(shortCtx, scw.sourceTablets[shardIndex])
cancel()
if err != nil {
return err
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(uint32(shardIndex), status.Position, scw.maxTPS, throttler.ReplicationLagModuleDisabled, time.Now().Unix(), flags))
}
for _, si := range scw.destinationShards {
destinationWaitGroup.Add(1)
go func(keyspace, shard string) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Making and populating blp_checkpoint table")
keyspaceAndShard := topoproto.KeyspaceShardString(keyspace, shard)
if err := runSQLCommands(ctx, scw.wr, scw.tsc, keyspace, shard, scw.destinationDbNames[keyspaceAndShard], queries); err != nil {
processError("blp_checkpoint queries failed: %v", err)
}
}(si.Keyspace(), si.ShardName())
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
// TODO(alainjobart) this is a superset, some shards may not
// overlap, have to deal with this better (for N -> M splits
// where both N>1 and M>1)
if scw.strategy.skipSetSourceShards {
scw.wr.Logger().Infof("Skipping setting SourceShard on destination shards.")
} else {
for _, si := range scw.destinationShards {
scw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", si.Keyspace(), si.ShardName())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := scw.wr.SetSourceShards(shortCtx, si.Keyspace(), si.ShardName(), scw.sourceAliases, nil)
cancel()
if err != nil {
return fmt.Errorf("failed to set source shards: %v", err)
}
}
}
err = scw.findRefreshTargets(ctx)
if err != nil {
return fmt.Errorf("failed before refreshing state on destination tablets: %v", err)
}
// And force a state refresh (re-read topo) on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for shardIndex := range scw.destinationShards {
for _, tabletAlias := range scw.refreshAliases[shardIndex] {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Refreshing state on tablet %v", ti.AliasString())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := scw.wr.TabletManagerClient().RefreshState(shortCtx, ti.Tablet)
cancel()
if err != nil {
processError("RefreshState failed on tablet %v: %v", ti.AliasString(), err)
}
}(scw.refreshTablets[shardIndex][*tabletAlias])
}
}
destinationWaitGroup.Wait()
return firstError
}
// copy phase:
// - copy the data from source tablets to destination masters (wtih replication on)
// Assumes that the schema has already been created on each destination tablet
// (probably from vtctl's CopySchemaShard)
func (vscw *VerticalSplitCloneWorker) copy(ctx context.Context) error {
vscw.setState(WorkerStateCopy)
// get source schema
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sourceSchemaDefinition, err := vscw.wr.GetSchema(shortCtx, vscw.sourceAlias, vscw.tables, nil, true)
cancel()
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", topo.TabletAliasString(vscw.sourceAlias), err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter")
}
vscw.wr.Logger().Infof("Source tablet has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
vscw.Mu.Lock()
vscw.tableStatus = make([]*tableStatus, len(sourceSchemaDefinition.TableDefinitions))
for i, td := range sourceSchemaDefinition.TableDefinitions {
vscw.tableStatus[i] = &tableStatus{
name: td.Name,
rowCount: td.RowCount,
}
}
vscw.startTime = time.Now()
vscw.Mu.Unlock()
// Count rows
for i, td := range sourceSchemaDefinition.TableDefinitions {
vscw.tableStatus[i].mu.Lock()
if td.Type == myproto.TableBaseTable {
vscw.tableStatus[i].rowCount = td.RowCount
} else {
vscw.tableStatus[i].isView = true
}
vscw.tableStatus[i].mu.Unlock()
}
// In parallel, setup the channels to send SQL data chunks to
// for each destination tablet.
//
// mu protects firstError
mu := sync.Mutex{}
var firstError error
ctx, cancel = context.WithCancel(ctx)
processError := func(format string, args ...interface{}) {
vscw.wr.Logger().Errorf(format, args...)
mu.Lock()
if firstError == nil {
firstError = fmt.Errorf(format, args...)
cancel()
}
mu.Unlock()
}
destinationWaitGroup := sync.WaitGroup{}
// we create one channel for the destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannel := make(chan string, vscw.destinationWriterCount*2)
go func(shardName string, insertChannel chan string) {
for j := 0; j < vscw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func() {
defer destinationWaitGroup.Done()
if err := executeFetchLoop(ctx, vscw.wr, vscw, shardName, insertChannel); err != nil {
processError("executeFetchLoop failed: %v", err)
}
}()
}
}(vscw.destinationShard, insertChannel)
// Now for each table, read data chunks and send them to insertChannel
sourceWaitGroup := sync.WaitGroup{}
sema := sync2.NewSemaphore(vscw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TableView {
continue
}
chunks, err := FindChunks(ctx, vscw.wr, vscw.sourceTablet, td, vscw.minTableSizeForSplit, vscw.sourceReaderCount)
if err != nil {
return err
}
vscw.tableStatus[tableIndex].setThreadCount(len(chunks) - 1)
for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ {
sourceWaitGroup.Add(1)
go func(td *myproto.TableDefinition, tableIndex, chunkIndex int) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
vscw.tableStatus[tableIndex].threadStarted()
// build the query, and start the streaming
selectSQL := buildSQLFromChunks(vscw.wr, td, chunks, chunkIndex, topo.TabletAliasString(vscw.sourceAlias))
qrr, err := NewQueryResultReaderForTablet(ctx, vscw.wr.TopoServer(), vscw.sourceAlias, selectSQL)
if err != nil {
processError("NewQueryResultReaderForTablet failed: %v", err)
return
}
defer qrr.Close()
// process the data
if err := vscw.processData(td, tableIndex, qrr, insertChannel, vscw.destinationPackCount, ctx.Done()); err != nil {
processError("QueryResultReader failed: %v", err)
}
vscw.tableStatus[tableIndex].threadDone()
}(td, tableIndex, chunkIndex)
}
}
sourceWaitGroup.Wait()
close(insertChannel)
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
// then create and populate the blp_checkpoint table
if vscw.strategy.PopulateBlpCheckpoint {
// get the current position from the source
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
status, err := vscw.wr.TabletManagerClient().SlaveStatus(shortCtx, vscw.sourceTablet)
cancel()
if err != nil {
return err
}
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if vscw.strategy.DontStartBinlogPlayer {
flags = binlogplayer.BlpFlagDontStart
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, time.Now().Unix(), flags))
destinationWaitGroup.Add(1)
go func(shardName string) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Making and populating blp_checkpoint table")
if err := runSQLCommands(ctx, vscw.wr, vscw, shardName, queries); err != nil {
processError("blp_checkpoint queries failed: %v", err)
}
}(vscw.destinationShard)
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
if vscw.strategy.SkipSetSourceShards {
vscw.wr.Logger().Infof("Skipping setting SourceShard on destination shard.")
} else {
vscw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", vscw.destinationKeyspace, vscw.destinationShard)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := vscw.wr.SetSourceShards(shortCtx, vscw.destinationKeyspace, vscw.destinationShard, []*pb.TabletAlias{vscw.sourceAlias}, vscw.tables)
cancel()
if err != nil {
return fmt.Errorf("Failed to set source shards: %v", err)
}
}
err = vscw.findReloadTargets(ctx)
if err != nil {
return fmt.Errorf("failed before reloading schema on destination tablets: %v", err)
}
// And force a schema reload on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for _, tabletAlias := range vscw.reloadAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
vscw.wr.Logger().Infof("Reloading schema on tablet %v", ti.AliasString())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := vscw.wr.TabletManagerClient().ReloadSchema(shortCtx, ti)
cancel()
if err != nil {
processError("ReloadSchema failed on tablet %v: %v", ti.AliasString(), err)
}
}(vscw.reloadTablets[*tabletAlias])
}
destinationWaitGroup.Wait()
return firstError
}
func (vsdw *VerticalSplitDiffWorker) diff(ctx context.Context) error {
vsdw.SetState(WorkerStateDiff)
vsdw.wr.Logger().Infof("Gathering schema information...")
wg := sync.WaitGroup{}
rec := &concurrency.AllErrorRecorder{}
wg.Add(1)
go func() {
var err error
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
vsdw.destinationSchemaDefinition, err = vsdw.wr.GetSchema(
shortCtx, vsdw.destinationAlias, vsdw.shardInfo.SourceShards[0].Tables, nil /* excludeTables */, false /* includeViews */)
cancel()
rec.RecordError(err)
vsdw.wr.Logger().Infof("Got schema from destination %v", topoproto.TabletAliasString(vsdw.destinationAlias))
wg.Done()
}()
wg.Add(1)
go func() {
var err error
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
vsdw.sourceSchemaDefinition, err = vsdw.wr.GetSchema(
shortCtx, vsdw.sourceAlias, vsdw.shardInfo.SourceShards[0].Tables, nil /* excludeTables */, false /* includeViews */)
cancel()
rec.RecordError(err)
vsdw.wr.Logger().Infof("Got schema from source %v", topoproto.TabletAliasString(vsdw.sourceAlias))
wg.Done()
}()
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// Check the schema
vsdw.wr.Logger().Infof("Diffing the schema...")
rec = &concurrency.AllErrorRecorder{}
tmutils.DiffSchema("destination", vsdw.destinationSchemaDefinition, "source", vsdw.sourceSchemaDefinition, rec)
if rec.HasErrors() {
vsdw.wr.Logger().Warningf("Different schemas: %v", rec.Error())
} else {
vsdw.wr.Logger().Infof("Schema match, good.")
}
// run the diffs, 8 at a time
vsdw.wr.Logger().Infof("Running the diffs...")
sem := sync2.NewSemaphore(8, 0)
for _, tableDefinition := range vsdw.destinationSchemaDefinition.TableDefinitions {
wg.Add(1)
go func(tableDefinition *tabletmanagerdatapb.TableDefinition) {
defer wg.Done()
sem.Acquire()
defer sem.Release()
vsdw.wr.Logger().Infof("Starting the diff on table %v", tableDefinition.Name)
sourceQueryResultReader, err := TableScan(ctx, vsdw.wr.Logger(), vsdw.wr.TopoServer(), vsdw.sourceAlias, tableDefinition)
if err != nil {
newErr := fmt.Errorf("TableScan(source) failed: %v", err)
rec.RecordError(newErr)
vsdw.wr.Logger().Errorf("%v", newErr)
return
}
defer sourceQueryResultReader.Close()
destinationQueryResultReader, err := TableScan(ctx, vsdw.wr.Logger(), vsdw.wr.TopoServer(), vsdw.destinationAlias, tableDefinition)
if err != nil {
newErr := fmt.Errorf("TableScan(destination) failed: %v", err)
rec.RecordError(newErr)
vsdw.wr.Logger().Errorf("%v", newErr)
return
}
defer destinationQueryResultReader.Close()
differ, err := NewRowDiffer(sourceQueryResultReader, destinationQueryResultReader, tableDefinition)
if err != nil {
newErr := fmt.Errorf("NewRowDiffer() failed: %v", err)
rec.RecordError(newErr)
vsdw.wr.Logger().Errorf("%v", newErr)
return
}
report, err := differ.Go(vsdw.wr.Logger())
if err != nil {
vsdw.wr.Logger().Errorf("Differ.Go failed: %v", err)
} else {
if report.HasDifferences() {
err := fmt.Errorf("Table %v has differences: %v", tableDefinition.Name, report.String())
rec.RecordError(err)
vsdw.wr.Logger().Errorf("%v", err)
} else {
vsdw.wr.Logger().Infof("Table %v checks out (%v rows processed, %v qps)", tableDefinition.Name, report.processedRows, report.processingQPS)
}
}
}(tableDefinition)
}
wg.Wait()
return rec.Error()
}
// copy phase:
// - copy the data from source tablets to destination masters (with replication on)
// Assumes that the schema has already been created on each destination tablet
// (probably from vtctl's CopySchemaShard)
func (scw *SplitCloneWorker) clone(ctx context.Context, state StatusWorkerState) error {
if state != WorkerStateCloneOnline && state != WorkerStateCloneOffline {
panic(fmt.Sprintf("invalid state passed to clone(): %v", state))
}
scw.setState(state)
start := time.Now()
defer func() {
statsStateDurationsNs.Set(string(state), time.Now().Sub(start).Nanoseconds())
}()
var firstSourceTablet *topodatapb.Tablet
if state == WorkerStateCloneOffline {
// Use the first source tablet which we took offline.
firstSourceTablet = scw.sourceTablets[0]
} else {
// Pick any healthy serving source tablet.
si := scw.sourceShards[0]
tablets := scw.tsc.GetTabletStats(si.Keyspace(), si.ShardName(), topodatapb.TabletType_RDONLY)
if len(tablets) == 0 {
// We fail fast on this problem and don't retry because at the start all tablets should be healthy.
return fmt.Errorf("no healthy RDONLY tablet in source shard (%v) available (required to find out the schema)", topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName()))
}
firstSourceTablet = tablets[0].Tablet
}
var statsCounters []*stats.Counters
var tableStatusList *tableStatusList
switch state {
case WorkerStateCloneOnline:
statsCounters = []*stats.Counters{statsOnlineInsertsCounters, statsOnlineUpdatesCounters, statsOnlineDeletesCounters, statsOnlineEqualRowsCounters}
tableStatusList = scw.tableStatusListOnline
case WorkerStateCloneOffline:
statsCounters = []*stats.Counters{statsOfflineInsertsCounters, statsOfflineUpdatesCounters, statsOfflineDeletesCounters, statsOfflineEqualRowsCounters}
tableStatusList = scw.tableStatusListOffline
}
// The throttlers exist only for the duration of this clone() call.
// That means a SplitClone invocation with both online and offline phases
// will create throttlers for each phase.
if err := scw.createThrottlers(); err != nil {
return err
}
defer scw.closeThrottlers()
sourceSchemaDefinition, err := scw.getSourceSchema(ctx, firstSourceTablet)
if err != nil {
return err
}
scw.wr.Logger().Infof("Source tablet 0 has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
tableStatusList.initialize(sourceSchemaDefinition)
// In parallel, setup the channels to send SQL data chunks to for each destination tablet:
//
// mu protects the context for cancelation, and firstError
mu := sync.Mutex{}
var firstError error
ctx, cancelCopy := context.WithCancel(ctx)
processError := func(format string, args ...interface{}) {
scw.wr.Logger().Errorf(format, args...)
mu.Lock()
if firstError == nil {
firstError = fmt.Errorf(format, args...)
cancelCopy()
}
mu.Unlock()
}
insertChannels := make([]chan string, len(scw.destinationShards))
destinationWaitGroup := sync.WaitGroup{}
for shardIndex, si := range scw.destinationShards {
// We create one channel per destination tablet. It is sized to have a
// buffer of a maximum of destinationWriterCount * 2 items, to hopefully
// always have data. We then have destinationWriterCount go routines reading
// from it.
insertChannels[shardIndex] = make(chan string, scw.destinationWriterCount*2)
for j := 0; j < scw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func(keyspace, shard string, insertChannel chan string, throttler *throttler.Throttler, threadID int) {
defer destinationWaitGroup.Done()
defer throttler.ThreadFinished(threadID)
executor := newExecutor(scw.wr, scw.tsc, throttler, keyspace, shard, threadID)
if err := executor.fetchLoop(ctx, insertChannel); err != nil {
processError("executer.FetchLoop failed: %v", err)
}
}(si.Keyspace(), si.ShardName(), insertChannels[shardIndex], scw.getThrottler(si.Keyspace(), si.ShardName()), j)
}
}
// Now for each table, read data chunks and send them to all
// insertChannels
sourceWaitGroup := sync.WaitGroup{}
sema := sync2.NewSemaphore(scw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
td = reorderColumnsPrimaryKeyFirst(td)
keyResolver, err := scw.createKeyResolver(td)
if err != nil {
return fmt.Errorf("cannot resolve sharding keys for keyspace %v: %v", scw.destinationKeyspace, err)
}
// TODO(mberlin): We're going to chunk *all* source shards based on the MIN
// and MAX values of the *first* source shard. Is this going to be a problem?
chunks, err := generateChunks(ctx, scw.wr, firstSourceTablet, td, scw.chunkCount, scw.minRowsPerChunk)
if err != nil {
return err
}
tableStatusList.setThreadCount(tableIndex, len(chunks))
for _, c := range chunks {
sourceWaitGroup.Add(1)
go func(td *tabletmanagerdatapb.TableDefinition, tableIndex int, chunk chunk) {
defer sourceWaitGroup.Done()
errPrefix := fmt.Sprintf("table=%v chunk=%v", td.Name, chunk)
// We need our own error per Go routine to avoid races.
var err error
sema.Acquire()
defer sema.Release()
tableStatusList.threadStarted(tableIndex)
if state == WorkerStateCloneOnline {
// Wait for enough healthy tablets (they might have become unhealthy
// and their replication lag might have increased since we started.)
if err := scw.waitForTablets(ctx, scw.sourceShards, *retryDuration); err != nil {
processError("%v: No healthy source tablets found (gave up after %v): ", errPrefix, *retryDuration, err)
return
}
}
// Set up readers for the diff. There will be one reader for every
// source and destination shard.
sourceReaders := make([]ResultReader, len(scw.sourceShards))
destReaders := make([]ResultReader, len(scw.destinationShards))
for shardIndex, si := range scw.sourceShards {
var tp tabletProvider
allowMultipleRetries := true
if state == WorkerStateCloneOffline {
tp = newSingleTabletProvider(ctx, scw.wr.TopoServer(), scw.offlineSourceAliases[shardIndex])
// allowMultipleRetries is false to avoid that we'll keep retrying
// on the same tablet alias for hours. This guards us against the
// situation that an offline tablet gets restarted and serves again.
// In that case we cannot use it because its replication is no
// longer stopped at the same point as we took it offline initially.
allowMultipleRetries = false
} else {
tp = newShardTabletProvider(scw.tsc, scw.tabletTracker, si.Keyspace(), si.ShardName())
}
sourceResultReader, err := NewRestartableResultReader(ctx, scw.wr.Logger(), tp, td, chunk, allowMultipleRetries)
if err != nil {
processError("%v: NewRestartableResultReader for source: %v failed", errPrefix, tp.description())
return
}
defer sourceResultReader.Close()
sourceReaders[shardIndex] = sourceResultReader
}
// Wait for enough healthy tablets (they might have become unhealthy
// and their replication lag might have increased due to a previous
// chunk pipeline.)
if err := scw.waitForTablets(ctx, scw.destinationShards, *retryDuration); err != nil {
processError("%v: No healthy destination tablets found (gave up after %v): ", errPrefix, *retryDuration, err)
return
}
for shardIndex, si := range scw.destinationShards {
tp := newShardTabletProvider(scw.tsc, scw.tabletTracker, si.Keyspace(), si.ShardName())
destResultReader, err := NewRestartableResultReader(ctx, scw.wr.Logger(), tp, td, chunk, true /* allowMultipleRetries */)
if err != nil {
processError("%v: NewRestartableResultReader for destination: %v failed: %v", errPrefix, tp.description(), err)
return
}
defer destResultReader.Close()
destReaders[shardIndex] = destResultReader
}
var sourceReader ResultReader
var destReader ResultReader
if len(sourceReaders) >= 2 {
sourceReader, err = NewResultMerger(sourceReaders, len(td.PrimaryKeyColumns))
if err != nil {
processError("%v: NewResultMerger for source tablets failed: %v", errPrefix, err)
return
}
} else {
sourceReader = sourceReaders[0]
}
if len(destReaders) >= 2 {
destReader, err = NewResultMerger(destReaders, len(td.PrimaryKeyColumns))
if err != nil {
processError("%v: NewResultMerger for destination tablets failed: %v", errPrefix, err)
return
}
} else {
destReader = destReaders[0]
}
dbNames := make([]string, len(scw.destinationShards))
for i, si := range scw.destinationShards {
keyspaceAndShard := topoproto.KeyspaceShardString(si.Keyspace(), si.ShardName())
dbNames[i] = scw.destinationDbNames[keyspaceAndShard]
}
// Compare the data and reconcile any differences.
differ, err := NewRowDiffer2(ctx, sourceReader, destReader, td, tableStatusList, tableIndex,
scw.destinationShards, keyResolver,
insertChannels, ctx.Done(), dbNames, scw.writeQueryMaxRows, scw.writeQueryMaxSize, scw.writeQueryMaxRowsDelete, statsCounters)
if err != nil {
processError("%v: NewRowDiffer2 failed: %v", errPrefix, err)
return
}
// Ignore the diff report because all diffs should get reconciled.
_ /* DiffReport */, err = differ.Diff()
if err != nil {
processError("%v: RowDiffer2 failed: %v", errPrefix, err)
return
}
tableStatusList.threadDone(tableIndex)
}(td, tableIndex, c)
}
}
sourceWaitGroup.Wait()
for shardIndex := range scw.destinationShards {
close(insertChannels[shardIndex])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
if state == WorkerStateCloneOffline {
// Create and populate the blp_checkpoint table to give filtered replication
// a starting point.
if scw.strategy.skipPopulateBlpCheckpoint {
scw.wr.Logger().Infof("Skipping populating the blp_checkpoint table")
} else {
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if scw.strategy.dontStartBinlogPlayer {
flags = binlogplayer.BlpFlagDontStart
}
// get the current position from the sources
for shardIndex := range scw.sourceShards {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
status, err := scw.wr.TabletManagerClient().SlaveStatus(shortCtx, scw.sourceTablets[shardIndex])
cancel()
if err != nil {
return err
}
// TODO(mberlin): Fill in scw.maxReplicationLag once the adapative
// throttler is enabled by default.
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(uint32(shardIndex), status.Position, scw.maxTPS, throttler.ReplicationLagModuleDisabled, time.Now().Unix(), flags))
}
for _, si := range scw.destinationShards {
destinationWaitGroup.Add(1)
go func(keyspace, shard string) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Making and populating blp_checkpoint table")
keyspaceAndShard := topoproto.KeyspaceShardString(keyspace, shard)
if err := runSQLCommands(ctx, scw.wr, scw.tsc, keyspace, shard, scw.destinationDbNames[keyspaceAndShard], queries); err != nil {
processError("blp_checkpoint queries failed: %v", err)
}
}(si.Keyspace(), si.ShardName())
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Configure filtered replication by setting the SourceShard info.
// The master tablets won't enable filtered replication (the binlog player)
// until they re-read the topology due to a restart or a reload.
// TODO(alainjobart) this is a superset, some shards may not
// overlap, have to deal with this better (for N -> M splits
// where both N>1 and M>1)
if scw.strategy.skipSetSourceShards {
scw.wr.Logger().Infof("Skipping setting SourceShard on destination shards.")
} else {
for _, si := range scw.destinationShards {
scw.wr.Logger().Infof("Setting SourceShard on shard %v/%v (tables: %v)", si.Keyspace(), si.ShardName(), scw.tables)
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := scw.wr.SetSourceShards(shortCtx, si.Keyspace(), si.ShardName(), scw.offlineSourceAliases, scw.tables)
cancel()
if err != nil {
return fmt.Errorf("failed to set source shards: %v", err)
}
}
}
// Force a state refresh (re-read topo) on all destination tablets.
// The master tablet will end up starting filtered replication at this point.
//
// Find all tablets first, then refresh the state on each in parallel.
err = scw.findRefreshTargets(ctx)
if err != nil {
return fmt.Errorf("failed before refreshing state on destination tablets: %v", err)
}
for shardIndex := range scw.destinationShards {
for _, tabletAlias := range scw.refreshAliases[shardIndex] {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
scw.wr.Logger().Infof("Refreshing state on tablet %v", ti.AliasString())
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
err := scw.wr.TabletManagerClient().RefreshState(shortCtx, ti.Tablet)
cancel()
if err != nil {
processError("RefreshState failed on tablet %v: %v", ti.AliasString(), err)
}
}(scw.refreshTablets[shardIndex][*tabletAlias])
}
}
} // clonePhase == offline
destinationWaitGroup.Wait()
return firstError
}
func (sdw *SplitDiffWorker) diff(ctx context.Context) error {
sdw.SetState(WorkerStateDiff)
sdw.wr.Logger().Infof("Gathering schema information...")
wg := sync.WaitGroup{}
rec := &concurrency.AllErrorRecorder{}
wg.Add(1)
go func() {
var err error
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sdw.destinationSchemaDefinition, err = sdw.wr.GetSchema(
shortCtx, sdw.destinationAlias, nil /* tables */, sdw.excludeTables, false /* includeViews */)
cancel()
rec.RecordError(err)
sdw.wr.Logger().Infof("Got schema from destination %v", sdw.destinationAlias)
wg.Done()
}()
wg.Add(1)
go func() {
var err error
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sdw.sourceSchemaDefinition, err = sdw.wr.GetSchema(
shortCtx, sdw.sourceAlias, nil /* tables */, sdw.excludeTables, false /* includeViews */)
cancel()
rec.RecordError(err)
sdw.wr.Logger().Infof("Got schema from source %v", sdw.sourceAlias)
wg.Done()
}()
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
sdw.wr.Logger().Infof("Diffing the schema...")
rec = &concurrency.AllErrorRecorder{}
tmutils.DiffSchema("destination", sdw.destinationSchemaDefinition, "source", sdw.sourceSchemaDefinition, rec)
if rec.HasErrors() {
sdw.wr.Logger().Warningf("Different schemas: %v", rec.Error().Error())
} else {
sdw.wr.Logger().Infof("Schema match, good.")
}
// read the vschema if needed
var keyspaceSchema *vindexes.KeyspaceSchema
if *useV3ReshardingMode {
kschema, err := sdw.wr.TopoServer().GetVSchema(ctx, sdw.keyspace)
if err != nil {
return fmt.Errorf("cannot load VSchema for keyspace %v: %v", sdw.keyspace, err)
}
if kschema == nil {
return fmt.Errorf("no VSchema for keyspace %v", sdw.keyspace)
}
keyspaceSchema, err = vindexes.BuildKeyspaceSchema(kschema, sdw.keyspace)
if err != nil {
return fmt.Errorf("cannot build vschema for keyspace %v: %v", sdw.keyspace, err)
}
}
// Compute the overlap keyrange. Later, we'll compare it with
// source or destination keyrange. If it matches either,
// we'll just ask for all the data. If the overlap is a subset,
// we'll filter.
overlap, err := key.KeyRangesOverlap(sdw.shardInfo.KeyRange, sdw.shardInfo.SourceShards[sdw.sourceUID].KeyRange)
if err != nil {
return fmt.Errorf("Source shard doesn't overlap with destination: %v", err)
}
// run the diffs, 8 at a time
sdw.wr.Logger().Infof("Running the diffs...")
// TODO(mberlin): Parameterize the hard coded value 8.
sem := sync2.NewSemaphore(8, 0)
for _, tableDefinition := range sdw.destinationSchemaDefinition.TableDefinitions {
wg.Add(1)
go func(tableDefinition *tabletmanagerdatapb.TableDefinition) {
defer wg.Done()
sem.Acquire()
defer sem.Release()
sdw.wr.Logger().Infof("Starting the diff on table %v", tableDefinition.Name)
// On the source, see if we need a full scan
// or a filtered scan.
var sourceQueryResultReader *QueryResultReader
if key.KeyRangeEqual(overlap, sdw.shardInfo.SourceShards[sdw.sourceUID].KeyRange) {
sourceQueryResultReader, err = TableScan(ctx, sdw.wr.Logger(), sdw.wr.TopoServer(), sdw.sourceAlias, tableDefinition)
} else {
sourceQueryResultReader, err = TableScanByKeyRange(ctx, sdw.wr.Logger(), sdw.wr.TopoServer(), sdw.sourceAlias, tableDefinition, overlap, keyspaceSchema, sdw.keyspaceInfo.ShardingColumnName, sdw.keyspaceInfo.ShardingColumnType)
}
if err != nil {
newErr := fmt.Errorf("TableScan(ByKeyRange?)(source) failed: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf("%v", newErr)
return
}
defer sourceQueryResultReader.Close(ctx)
// On the destination, see if we need a full scan
// or a filtered scan.
var destinationQueryResultReader *QueryResultReader
if key.KeyRangeEqual(overlap, sdw.shardInfo.KeyRange) {
destinationQueryResultReader, err = TableScan(ctx, sdw.wr.Logger(), sdw.wr.TopoServer(), sdw.destinationAlias, tableDefinition)
} else {
destinationQueryResultReader, err = TableScanByKeyRange(ctx, sdw.wr.Logger(), sdw.wr.TopoServer(), sdw.destinationAlias, tableDefinition, overlap, keyspaceSchema, sdw.keyspaceInfo.ShardingColumnName, sdw.keyspaceInfo.ShardingColumnType)
}
if err != nil {
newErr := fmt.Errorf("TableScan(ByKeyRange?)(destination) failed: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf("%v", newErr)
return
}
defer destinationQueryResultReader.Close(ctx)
// Create the row differ.
differ, err := NewRowDiffer(sourceQueryResultReader, destinationQueryResultReader, tableDefinition)
if err != nil {
newErr := fmt.Errorf("NewRowDiffer() failed: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf("%v", newErr)
return
}
// And run the diff.
report, err := differ.Go(sdw.wr.Logger())
if err != nil {
newErr := fmt.Errorf("Differ.Go failed: %v", err.Error())
rec.RecordError(newErr)
sdw.wr.Logger().Errorf("%v", newErr)
} else {
if report.HasDifferences() {
err := fmt.Errorf("Table %v has differences: %v", tableDefinition.Name, report.String())
rec.RecordError(err)
sdw.wr.Logger().Warningf(err.Error())
} else {
sdw.wr.Logger().Infof("Table %v checks out (%v rows processed, %v qps)", tableDefinition.Name, report.processedRows, report.processingQPS)
}
}
}(tableDefinition)
}
wg.Wait()
return rec.Error()
}
func NewResourceConstraint(max int) *ResourceConstraint {
return &ResourceConstraint{semaphore: sync2.NewSemaphore(max, 0)}
}
"github.com/youtube/vitess/go/streamlog"
"github.com/youtube/vitess/go/sync2"
"github.com/youtube/vitess/go/vt/dbconfigs"
"github.com/youtube/vitess/go/vt/mysqlctl"
"github.com/youtube/vitess/go/vt/tabletserver/proto"
"github.com/youtube/vitess/go/vt/tabletserver/queryservice"
"golang.org/x/net/context"
pb "github.com/youtube/vitess/go/vt/proto/query"
)
var (
queryLogHandler = flag.String("query-log-stream-handler", "/debug/querylog", "URL handler for streaming queries log")
txLogHandler = flag.String("transaction-log-stream-handler", "/debug/txlog", "URL handler for streaming transactions log")
checkMySLQThrottler = sync2.NewSemaphore(1, 0)
)
func init() {
flag.IntVar(&qsConfig.PoolSize, "queryserver-config-pool-size", DefaultQsConfig.PoolSize, "query server connection pool size, connection pool is used by regular queries (non streaming, not in a transaction)")
flag.IntVar(&qsConfig.StreamPoolSize, "queryserver-config-stream-pool-size", DefaultQsConfig.StreamPoolSize, "query server stream pool size, stream pool is used by stream queries: queries that return results to client in a streaming fashion")
flag.IntVar(&qsConfig.TransactionCap, "queryserver-config-transaction-cap", DefaultQsConfig.TransactionCap, "query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout)")
flag.Float64Var(&qsConfig.TransactionTimeout, "queryserver-config-transaction-timeout", DefaultQsConfig.TransactionTimeout, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value")
flag.IntVar(&qsConfig.MaxResultSize, "queryserver-config-max-result-size", DefaultQsConfig.MaxResultSize, "query server max result size, maximum number of rows allowed to return from vttablet for non-streaming queries.")
flag.IntVar(&qsConfig.MaxDMLRows, "queryserver-config-max-dml-rows", DefaultQsConfig.MaxDMLRows, "query server max dml rows per statement, maximum number of rows allowed to return at a time for an upadte or delete with either 1) an equality where clauses on primary keys, or 2) a subselect statement. For update and delete statements in above two categories, vttablet will split the original query into multiple small queries based on this configuration value. ")
flag.IntVar(&qsConfig.StreamBufferSize, "queryserver-config-stream-buffer-size", DefaultQsConfig.StreamBufferSize, "query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call.")
flag.IntVar(&qsConfig.QueryCacheSize, "queryserver-config-query-cache-size", DefaultQsConfig.QueryCacheSize, "query server query cache size, maximum number of queries to be cached. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.")
flag.Float64Var(&qsConfig.SchemaReloadTime, "queryserver-config-schema-reload-time", DefaultQsConfig.SchemaReloadTime, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.")
flag.Float64Var(&qsConfig.QueryTimeout, "queryserver-config-query-timeout", DefaultQsConfig.QueryTimeout, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.")
flag.Float64Var(&qsConfig.TxPoolTimeout, "queryserver-config-txpool-timeout", DefaultQsConfig.TxPoolTimeout, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full")
flag.Float64Var(&qsConfig.IdleTimeout, "queryserver-config-idle-timeout", DefaultQsConfig.IdleTimeout, "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.")
func (sdw *SplitDiffWorker) diff(ctx context.Context) error {
sdw.SetState(WorkerStateDiff)
sdw.wr.Logger().Infof("Gathering schema information...")
sdw.sourceSchemaDefinitions = make([]*myproto.SchemaDefinition, len(sdw.sourceAliases))
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
wg.Add(1)
go func() {
var err error
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sdw.destinationSchemaDefinition, err = sdw.wr.GetSchema(
shortCtx, sdw.destinationAlias, nil /* tables */, sdw.excludeTables, false /* includeViews */)
cancel()
rec.RecordError(err)
sdw.wr.Logger().Infof("Got schema from destination %v", sdw.destinationAlias)
wg.Done()
}()
for i, sourceAlias := range sdw.sourceAliases {
wg.Add(1)
go func(i int, sourceAlias pb.TabletAlias) {
var err error
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
sdw.sourceSchemaDefinitions[i], err = sdw.wr.GetSchema(
shortCtx, &sourceAlias, nil /* tables */, sdw.excludeTables, false /* includeViews */)
cancel()
rec.RecordError(err)
sdw.wr.Logger().Infof("Got schema from source[%v] %v", i, sourceAlias)
wg.Done()
}(i, *sourceAlias)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// TODO(alainjobart) Checking against each source may be
// overkill, if all sources have the same schema?
sdw.wr.Logger().Infof("Diffing the schema...")
rec = concurrency.AllErrorRecorder{}
for i, sourceSchemaDefinition := range sdw.sourceSchemaDefinitions {
sourceName := fmt.Sprintf("source[%v]", i)
myproto.DiffSchema("destination", sdw.destinationSchemaDefinition, sourceName, sourceSchemaDefinition, &rec)
}
if rec.HasErrors() {
sdw.wr.Logger().Warningf("Different schemas: %v", rec.Error().Error())
} else {
sdw.wr.Logger().Infof("Schema match, good.")
}
// run the diffs, 8 at a time
sdw.wr.Logger().Infof("Running the diffs...")
sem := sync2.NewSemaphore(8, 0)
for _, tableDefinition := range sdw.destinationSchemaDefinition.TableDefinitions {
wg.Add(1)
go func(tableDefinition *myproto.TableDefinition) {
defer wg.Done()
sem.Acquire()
defer sem.Release()
sdw.wr.Logger().Infof("Starting the diff on table %v", tableDefinition.Name)
if len(sdw.sourceAliases) != 1 {
err := fmt.Errorf("Don't support more than one source for table yet: %v", tableDefinition.Name)
rec.RecordError(err)
sdw.wr.Logger().Errorf(err.Error())
return
}
overlap, err := key.KeyRangesOverlap(sdw.shardInfo.KeyRange, sdw.shardInfo.SourceShards[0].KeyRange)
if err != nil {
newErr := fmt.Errorf("Source shard doesn't overlap with destination????: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf(newErr.Error())
return
}
sourceQueryResultReader, err := TableScanByKeyRange(ctx, sdw.wr.Logger(), sdw.wr.TopoServer(), sdw.sourceAliases[0], tableDefinition, overlap, key.ProtoToKeyspaceIdType(sdw.keyspaceInfo.ShardingColumnType))
if err != nil {
newErr := fmt.Errorf("TableScanByKeyRange(source) failed: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf(newErr.Error())
return
}
defer sourceQueryResultReader.Close()
destinationQueryResultReader, err := TableScanByKeyRange(ctx, sdw.wr.Logger(), sdw.wr.TopoServer(), sdw.destinationAlias, tableDefinition, nil, key.ProtoToKeyspaceIdType(sdw.keyspaceInfo.ShardingColumnType))
if err != nil {
newErr := fmt.Errorf("TableScanByKeyRange(destination) failed: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf(newErr.Error())
return
}
defer destinationQueryResultReader.Close()
differ, err := NewRowDiffer(sourceQueryResultReader, destinationQueryResultReader, tableDefinition)
if err != nil {
newErr := fmt.Errorf("NewRowDiffer() failed: %v", err)
rec.RecordError(newErr)
sdw.wr.Logger().Errorf(newErr.Error())
return
}
report, err := differ.Go(sdw.wr.Logger())
if err != nil {
newErr := fmt.Errorf("Differ.Go failed: %v", err.Error())
rec.RecordError(newErr)
sdw.wr.Logger().Errorf(newErr.Error())
} else {
if report.HasDifferences() {
err := fmt.Errorf("Table %v has differences: %v", tableDefinition.Name, report.String())
rec.RecordError(err)
sdw.wr.Logger().Warningf(err.Error())
} else {
sdw.wr.Logger().Infof("Table %v checks out (%v rows processed, %v qps)", tableDefinition.Name, report.processedRows, report.processingQPS)
}
}
}(tableDefinition)
}
wg.Wait()
return rec.Error()
}
// copy phase:
// - get schema on the source, filter tables
// - create tables on all destinations
// - copy the data
func (vscw *VerticalSplitCloneWorker) copy() error {
vscw.setState(stateVSCCopy)
// get source schema
sourceSchemaDefinition, err := vscw.wr.GetSchema(vscw.sourceAlias, vscw.tables, nil, true)
if err != nil {
return fmt.Errorf("cannot get schema from source %v: %v", vscw.sourceAlias, err)
}
if len(sourceSchemaDefinition.TableDefinitions) == 0 {
return fmt.Errorf("no tables matching the table filter")
}
log.Infof("Source tablet has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions))
vscw.mu.Lock()
vscw.tableStatus = make([]tableStatus, len(sourceSchemaDefinition.TableDefinitions))
for i, td := range sourceSchemaDefinition.TableDefinitions {
vscw.tableStatus[i].name = td.Name
vscw.tableStatus[i].rowCount = td.RowCount
}
vscw.startTime = time.Now()
vscw.mu.Unlock()
// Create all the commands to create the destination schema:
// - createDbCmds will create the database and the tables
// - createViewCmds will create the views
// - alterTablesCmds will modify the tables at the end if needed
// (all need template substitution for {{.DatabaseName}})
createDbCmds := make([]string, 0, len(sourceSchemaDefinition.TableDefinitions)+1)
createDbCmds = append(createDbCmds, sourceSchemaDefinition.DatabaseSchema)
createViewCmds := make([]string, 0, 16)
alterTablesCmds := make([]string, 0, 16)
for i, td := range sourceSchemaDefinition.TableDefinitions {
vscw.tableStatus[i].mu.Lock()
if td.Type == myproto.TABLE_BASE_TABLE {
create, alter, err := mysqlctl.MakeSplitCreateTableSql(td.Schema, "{{.DatabaseName}}", td.Name, vscw.strategy)
if err != nil {
return fmt.Errorf("MakeSplitCreateTableSql(%v) returned: %v", td.Name, err)
}
createDbCmds = append(createDbCmds, create)
if alter != "" {
alterTablesCmds = append(alterTablesCmds, alter)
}
vscw.tableStatus[i].state = "before table creation"
vscw.tableStatus[i].rowCount = td.RowCount
} else {
createViewCmds = append(createViewCmds, td.Schema)
vscw.tableStatus[i].state = "before view creation"
vscw.tableStatus[i].rowCount = 0
}
vscw.tableStatus[i].mu.Unlock()
}
// For each destination tablet (in parallel):
// - create the schema
// - setup the channels to send SQL data chunks
//
// mu protects the abort channel for closing, and firstError
mu := sync.Mutex{}
abort := make(chan struct{})
var firstError error
processError := func(format string, args ...interface{}) {
log.Errorf(format, args...)
mu.Lock()
if abort != nil {
close(abort)
abort = nil
firstError = fmt.Errorf(format, args...)
}
mu.Unlock()
}
insertChannels := make([]chan string, len(vscw.destinationAliases))
destinationWaitGroup := sync.WaitGroup{}
for i, tabletAlias := range vscw.destinationAliases {
// we create one channel per destination tablet. It
// is sized to have a buffer of a maximum of
// destinationWriterCount * 2 items, to hopefully
// always have data. We then have
// destinationWriterCount go routines reading from it.
insertChannels[i] = make(chan string, vscw.destinationWriterCount*2)
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo, insertChannel chan string) {
defer destinationWaitGroup.Done()
log.Infof("Creating tables on tablet %v", ti.Alias)
if err := vscw.runSqlCommands(ti, createDbCmds, abort); err != nil {
processError("createDbCmds failed: %v", err)
return
}
if len(createViewCmds) > 0 {
log.Infof("Creating views on tablet %v", ti.Alias)
if err := vscw.runSqlCommands(ti, createViewCmds, abort); err != nil {
processError("createViewCmds failed: %v", err)
return
}
}
for j := 0; j < vscw.destinationWriterCount; j++ {
destinationWaitGroup.Add(1)
go func() {
defer destinationWaitGroup.Done()
for {
select {
case cmd, ok := <-insertChannel:
if !ok {
return
}
cmd = "INSERT INTO `" + ti.DbName() + "`." + cmd
_, err := vscw.wr.ActionInitiator().ExecuteFetch(ti, cmd, 0, false, true, 30*time.Second)
if err != nil {
processError("ExecuteFetch failed: %v", err)
return
}
case <-abort:
return
}
}
}()
}
}(vscw.destinationTablets[tabletAlias], insertChannels[i])
}
// Now for each table, read data chunks and send them to all
// insertChannels
sourceWaitGroup := sync.WaitGroup{}
sema := sync2.NewSemaphore(vscw.sourceReaderCount, 0)
for tableIndex, td := range sourceSchemaDefinition.TableDefinitions {
if td.Type == myproto.TABLE_VIEW {
vscw.tableStatus[tableIndex].setState("view created")
continue
}
vscw.tableStatus[tableIndex].setState("before copy")
chunks, err := vscw.findChunks(vscw.sourceTablet, td)
if err != nil {
return err
}
for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ {
sourceWaitGroup.Add(1)
go func(td myproto.TableDefinition, tableIndex, chunkIndex int) {
defer sourceWaitGroup.Done()
sema.Acquire()
defer sema.Release()
vscw.tableStatus[tableIndex].setState("started the copy")
// build the query, and start the streaming
selectSQL := "SELECT " + strings.Join(td.Columns, ", ") + " FROM " + td.Name
if chunks[chunkIndex] != "" || chunks[chunkIndex+1] != "" {
log.Infof("Starting to stream all data from table %v between '%v' and '%v'", td.Name, chunks[chunkIndex], chunks[chunkIndex+1])
clauses := make([]string, 0, 2)
if chunks[chunkIndex] != "" {
clauses = append(clauses, td.PrimaryKeyColumns[0]+">="+chunks[chunkIndex])
}
if chunks[chunkIndex+1] != "" {
clauses = append(clauses, td.PrimaryKeyColumns[0]+"<"+chunks[chunkIndex+1])
}
selectSQL += " WHERE " + strings.Join(clauses, " AND ")
} else {
log.Infof("Starting to stream all data from table %v", td.Name)
}
if len(td.PrimaryKeyColumns) > 0 {
selectSQL += " ORDER BY " + strings.Join(td.PrimaryKeyColumns, ", ")
}
qrr, err := NewQueryResultReaderForTablet(vscw.wr.TopoServer(), vscw.sourceAlias, selectSQL)
if err != nil {
processError("NewQueryResultReaderForTablet failed: %v", err)
return
}
// process the data
baseCmd := td.Name + "(" + strings.Join(td.Columns, ", ") + ") VALUES "
loop:
for {
select {
case r, ok := <-qrr.Output:
if !ok {
if err := qrr.Error(); err != nil {
// error case
processError("QueryResultReader failed: %v", err)
return
}
// we're done with the data
break loop
}
// send the rows to be inserted
vscw.tableStatus[tableIndex].addCopiedRows(len(r.Rows))
cmd := baseCmd + makeValueString(qrr.Fields, r)
for _, c := range insertChannels {
c <- cmd
}
case <-abort:
return
}
}
}(td, tableIndex, chunkIndex)
}
}
sourceWaitGroup.Wait()
for _, c := range insertChannels {
close(c)
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
// do the post-copy alters if any
if len(alterTablesCmds) > 0 {
for _, tabletAlias := range vscw.destinationAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
log.Infof("Altering tables on tablet %v", ti.Alias)
if err := vscw.runSqlCommands(ti, alterTablesCmds, abort); err != nil {
processError("alterTablesCmds failed on tablet %v: %v", ti.Alias, err)
}
}(vscw.destinationTablets[tabletAlias])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// then create and populate the blp_checkpoint table
if strings.Index(vscw.strategy, "populateBlpCheckpoint") != -1 {
// get the current position from the source
pos, err := vscw.wr.ActionInitiator().SlavePosition(vscw.sourceTablet, 30*time.Second)
if err != nil {
return err
}
queries := make([]string, 0, 4)
queries = append(queries, binlogplayer.CreateBlpCheckpoint()...)
flags := ""
if strings.Index(vscw.strategy, "dontStartBinlogPlayer") != -1 {
flags = binlogplayer.BLP_FLAG_DONT_START
}
queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, pos.MasterLogGTIDField.Value, time.Now().Unix(), flags))
for _, tabletAlias := range vscw.destinationAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
log.Infof("Making and populating blp_checkpoint table on tablet %v", ti.Alias)
if err := vscw.runSqlCommands(ti, queries, abort); err != nil {
processError("blp_checkpoint queries failed on tablet %v: %v", ti.Alias, err)
}
}(vscw.destinationTablets[tabletAlias])
}
destinationWaitGroup.Wait()
if firstError != nil {
return firstError
}
}
// Now we're done with data copy, update the shard's source info.
log.Infof("Setting SourceShard on shard %v/%v", vscw.destinationKeyspace, vscw.destinationShard)
if err := vscw.wr.SetSourceShards(vscw.destinationKeyspace, vscw.destinationShard, []topo.TabletAlias{vscw.sourceAlias}, vscw.tables); err != nil {
return fmt.Errorf("Failed to set source shards: %v", err)
}
// And force a schema reload on all destination tablets.
// The master tablet will end up starting filtered replication
// at this point.
for _, tabletAlias := range vscw.destinationAliases {
destinationWaitGroup.Add(1)
go func(ti *topo.TabletInfo) {
defer destinationWaitGroup.Done()
log.Infof("Reloading schema on tablet %v", ti.Alias)
if err := vscw.wr.ActionInitiator().ReloadSchema(ti, 30*time.Second); err != nil {
processError("ReloadSchema failed on tablet %v: %v", ti.Alias, err)
}
}(vscw.destinationTablets[tabletAlias])
}
destinationWaitGroup.Wait()
return firstError
}