// copy phase: // - get schema on the sources, filter tables // - create tables on all destinations // - copy the data func (scw *SplitCloneWorker) copy() error { scw.setState(stateSCCopy) // get source schema from the first shard // TODO(alainjobart): for now, we assume the schema is compatible // on all source shards. Furthermore, we estimate the number of rows // in each source shard for each table to be about the same // (rowCount is used to estimate an ETA) sourceSchemaDefinition, err := scw.wr.GetSchema(scw.sourceAliases[0], nil, scw.excludeTables, true) if err != nil { return fmt.Errorf("cannot get schema from source %v: %v", scw.sourceAliases[0], err) } if len(sourceSchemaDefinition.TableDefinitions) == 0 { return fmt.Errorf("no tables matching the table filter in tablet %v", scw.sourceAliases[0]) } scw.wr.Logger().Infof("Source tablet 0 has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions)) scw.mu.Lock() scw.tableStatus = make([]tableStatus, len(sourceSchemaDefinition.TableDefinitions)) for i, td := range sourceSchemaDefinition.TableDefinitions { scw.tableStatus[i].name = td.Name scw.tableStatus[i].rowCount = td.RowCount * uint64(len(scw.sourceAliases)) } scw.startTime = time.Now() scw.mu.Unlock() // Create all the commands to create the destination schema: // - createDbCmds will create the database and the tables // - createViewCmds will create the views // - alterTablesCmds will modify the tables at the end if needed // (all need template substitution for {{.DatabaseName}}) createDbCmds := make([]string, 0, len(sourceSchemaDefinition.TableDefinitions)+1) createDbCmds = append(createDbCmds, sourceSchemaDefinition.DatabaseSchema) createViewCmds := make([]string, 0, 16) alterTablesCmds := make([]string, 0, 16) columnIndexes := make([]int, len(sourceSchemaDefinition.TableDefinitions)) for tableIndex, td := range sourceSchemaDefinition.TableDefinitions { if td.Type == myproto.TABLE_BASE_TABLE { // build the create and alter statements create, alter, err := mysqlctl.MakeSplitCreateTableSql(scw.wr.Logger(), td.Schema, "{{.DatabaseName}}", td.Name, scw.strategy) if err != nil { return fmt.Errorf("MakeSplitCreateTableSql(%v) returned: %v", td.Name, err) } createDbCmds = append(createDbCmds, create) if alter != "" { alterTablesCmds = append(alterTablesCmds, alter) } // find the column to split on columnIndexes[tableIndex] = -1 for i, name := range td.Columns { if name == scw.keyspaceInfo.ShardingColumnName { columnIndexes[tableIndex] = i break } } if columnIndexes[tableIndex] == -1 { return fmt.Errorf("table %v doesn't have a column named '%v'", td.Name, scw.keyspaceInfo.ShardingColumnName) } scw.tableStatus[tableIndex].mu.Lock() scw.tableStatus[tableIndex].state = "before table creation" scw.tableStatus[tableIndex].rowCount = td.RowCount scw.tableStatus[tableIndex].mu.Unlock() } else { scw.tableStatus[tableIndex].mu.Lock() createViewCmds = append(createViewCmds, td.Schema) scw.tableStatus[tableIndex].state = "before view creation" scw.tableStatus[tableIndex].rowCount = 0 scw.tableStatus[tableIndex].mu.Unlock() } } // For each destination tablet (in parallel): // - create the schema // - setup the channels to send SQL data chunks // // mu protects the abort channel for closing, and firstError mu := sync.Mutex{} abort := make(chan struct{}) var firstError error processError := func(format string, args ...interface{}) { scw.wr.Logger().Errorf(format, args...) mu.Lock() if abort != nil { close(abort) abort = nil firstError = fmt.Errorf(format, args...) } mu.Unlock() } insertChannels := make([][]chan string, len(scw.destinationShards)) destinationWaitGroup := sync.WaitGroup{} for shardIndex, _ := range scw.destinationShards { insertChannels[shardIndex] = make([]chan string, len(scw.destinationAliases[shardIndex])) for i, tabletAlias := range scw.destinationAliases[shardIndex] { // we create one channel per destination tablet. It // is sized to have a buffer of a maximum of // destinationWriterCount * 2 items, to hopefully // always have data. We then have // destinationWriterCount go routines reading from it. insertChannels[shardIndex][i] = make(chan string, scw.destinationWriterCount*2) destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo, insertChannel chan string) { defer destinationWaitGroup.Done() scw.wr.Logger().Infof("Creating tables on tablet %v", ti.Alias) if err := runSqlCommands(scw.wr, ti, createDbCmds, abort); err != nil { processError("createDbCmds failed: %v", err) return } if len(createViewCmds) > 0 { scw.wr.Logger().Infof("Creating views on tablet %v", ti.Alias) if err := runSqlCommands(scw.wr, ti, createViewCmds, abort); err != nil { processError("createViewCmds failed: %v", err) return } } for j := 0; j < scw.destinationWriterCount; j++ { destinationWaitGroup.Add(1) go func() { defer destinationWaitGroup.Done() if err := executeFetchLoop(scw.wr, ti, insertChannel, abort); err != nil { processError("executeFetchLoop failed: %v", err) } }() } }(scw.destinationTablets[shardIndex][tabletAlias], insertChannels[shardIndex][i]) } } // Now for each table, read data chunks and send them to all // insertChannels sourceWaitGroup := sync.WaitGroup{} for shardIndex, _ := range scw.sourceShards { sema := sync2.NewSemaphore(scw.sourceReaderCount, 0) for tableIndex, td := range sourceSchemaDefinition.TableDefinitions { if td.Type == myproto.TABLE_VIEW { continue } rowSplitter := NewRowSplitter(scw.destinationShards, scw.keyspaceInfo.ShardingColumnType, columnIndexes[tableIndex]) chunks, err := findChunks(scw.wr, scw.sourceTablets[shardIndex], td, scw.minTableSizeForSplit, scw.sourceReaderCount) if err != nil { return err } for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ { sourceWaitGroup.Add(1) go func(td *myproto.TableDefinition, tableIndex, chunkIndex int) { defer sourceWaitGroup.Done() sema.Acquire() defer sema.Release() // build the query, and start the streaming selectSQL := buildSQLFromChunks(scw.wr, td, chunks, chunkIndex, scw.sourceAliases[shardIndex].String()) qrr, err := NewQueryResultReaderForTablet(scw.wr.TopoServer(), scw.sourceAliases[shardIndex], selectSQL) if err != nil { processError("NewQueryResultReaderForTablet failed: %v", err) return } // process the data if err := scw.processData(td, tableIndex, qrr, rowSplitter, insertChannels, abort); err != nil { processError("processData failed: %v", err) } }(td, tableIndex, chunkIndex) } } } sourceWaitGroup.Wait() for shardIndex, _ := range scw.destinationShards { for _, c := range insertChannels[shardIndex] { close(c) } } destinationWaitGroup.Wait() if firstError != nil { return firstError } // do the post-copy alters if any if len(alterTablesCmds) > 0 { for shardIndex, _ := range scw.destinationShards { for _, tabletAlias := range scw.destinationAliases[shardIndex] { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() scw.wr.Logger().Infof("Altering tables on tablet %v", ti.Alias) if err := runSqlCommands(scw.wr, ti, alterTablesCmds, abort); err != nil { processError("alterTablesCmds failed on tablet %v: %v", ti.Alias, err) } }(scw.destinationTablets[shardIndex][tabletAlias]) } } destinationWaitGroup.Wait() if firstError != nil { return firstError } } // then create and populate the blp_checkpoint table if strings.Index(scw.strategy, "populateBlpCheckpoint") != -1 { queries := make([]string, 0, 4) queries = append(queries, binlogplayer.CreateBlpCheckpoint()...) flags := "" if strings.Index(scw.strategy, "dontStartBinlogPlayer") != -1 { flags = binlogplayer.BLP_FLAG_DONT_START } // get the current position from the sources for shardIndex, _ := range scw.sourceShards { status, err := scw.wr.TabletManagerClient().SlaveStatus(scw.sourceTablets[shardIndex], 30*time.Second) if err != nil { return err } queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, time.Now().Unix(), flags)) } for shardIndex, _ := range scw.destinationShards { for _, tabletAlias := range scw.destinationAliases[shardIndex] { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() scw.wr.Logger().Infof("Making and populating blp_checkpoint table on tablet %v", ti.Alias) if err := runSqlCommands(scw.wr, ti, queries, abort); err != nil { processError("blp_checkpoint queries failed on tablet %v: %v", ti.Alias, err) } }(scw.destinationTablets[shardIndex][tabletAlias]) } } destinationWaitGroup.Wait() if firstError != nil { return firstError } } // Now we're done with data copy, update the shard's source info. // TODO(alainjobart) this is a superset, some shards may not // overlap, have to deal with this better (for N -> M splits // where both N>1 and M>1) if strings.Index(scw.strategy, "skipSetSourceShards") != -1 { scw.wr.Logger().Infof("Skipping setting SourceShard on destination shards.") } else { for _, si := range scw.destinationShards { scw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", si.Keyspace(), si.ShardName()) if err := scw.wr.SetSourceShards(si.Keyspace(), si.ShardName(), scw.sourceAliases, nil); err != nil { return fmt.Errorf("Failed to set source shards: %v", err) } } } // And force a schema reload on all destination tablets. // The master tablet will end up starting filtered replication // at this point. for shardIndex, _ := range scw.destinationShards { for _, tabletAlias := range scw.destinationAliases[shardIndex] { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() scw.wr.Logger().Infof("Reloading schema on tablet %v", ti.Alias) if err := scw.wr.TabletManagerClient().ReloadSchema(ti, 30*time.Second); err != nil { processError("ReloadSchema failed on tablet %v: %v", ti.Alias, err) } }(scw.destinationTablets[shardIndex][tabletAlias]) } } destinationWaitGroup.Wait() return firstError }
// copy phase: // - get schema on the source, filter tables // - create tables on all destinations // - copy the data func (vscw *VerticalSplitCloneWorker) copy() error { vscw.setState(stateVSCCopy) // get source schema sourceSchemaDefinition, err := vscw.wr.GetSchema(vscw.sourceAlias, vscw.tables, nil, true) if err != nil { return fmt.Errorf("cannot get schema from source %v: %v", vscw.sourceAlias, err) } if len(sourceSchemaDefinition.TableDefinitions) == 0 { return fmt.Errorf("no tables matching the table filter") } vscw.wr.Logger().Infof("Source tablet has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions)) vscw.mu.Lock() vscw.tableStatus = make([]tableStatus, len(sourceSchemaDefinition.TableDefinitions)) for i, td := range sourceSchemaDefinition.TableDefinitions { vscw.tableStatus[i].name = td.Name vscw.tableStatus[i].rowCount = td.RowCount } vscw.startTime = time.Now() vscw.mu.Unlock() // Create all the commands to create the destination schema: // - createDbCmds will create the database and the tables // - createViewCmds will create the views // - alterTablesCmds will modify the tables at the end if needed // (all need template substitution for {{.DatabaseName}}) createDbCmds := make([]string, 0, len(sourceSchemaDefinition.TableDefinitions)+1) createDbCmds = append(createDbCmds, sourceSchemaDefinition.DatabaseSchema) createViewCmds := make([]string, 0, 16) alterTablesCmds := make([]string, 0, 16) for i, td := range sourceSchemaDefinition.TableDefinitions { vscw.tableStatus[i].mu.Lock() if td.Type == myproto.TABLE_BASE_TABLE { create, alter, err := mysqlctl.MakeSplitCreateTableSql(vscw.wr.Logger(), td.Schema, "{{.DatabaseName}}", td.Name, vscw.strategy) if err != nil { return fmt.Errorf("MakeSplitCreateTableSql(%v) returned: %v", td.Name, err) } createDbCmds = append(createDbCmds, create) if alter != "" { alterTablesCmds = append(alterTablesCmds, alter) } vscw.tableStatus[i].state = "before table creation" vscw.tableStatus[i].rowCount = td.RowCount } else { createViewCmds = append(createViewCmds, td.Schema) vscw.tableStatus[i].state = "before view creation" vscw.tableStatus[i].rowCount = 0 } vscw.tableStatus[i].mu.Unlock() } // For each destination tablet (in parallel): // - create the schema // - setup the channels to send SQL data chunks // // mu protects the abort channel for closing, and firstError mu := sync.Mutex{} abort := make(chan struct{}) var firstError error processError := func(format string, args ...interface{}) { vscw.wr.Logger().Errorf(format, args...) mu.Lock() if abort != nil { close(abort) abort = nil firstError = fmt.Errorf(format, args...) } mu.Unlock() } insertChannels := make([]chan string, len(vscw.destinationAliases)) destinationWaitGroup := sync.WaitGroup{} for i, tabletAlias := range vscw.destinationAliases { // we create one channel per destination tablet. It // is sized to have a buffer of a maximum of // destinationWriterCount * 2 items, to hopefully // always have data. We then have // destinationWriterCount go routines reading from it. insertChannels[i] = make(chan string, vscw.destinationWriterCount*2) destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo, insertChannel chan string) { defer destinationWaitGroup.Done() vscw.wr.Logger().Infof("Creating tables on tablet %v", ti.Alias) if err := runSqlCommands(vscw.wr, ti, createDbCmds, abort); err != nil { processError("createDbCmds failed: %v", err) return } if len(createViewCmds) > 0 { vscw.wr.Logger().Infof("Creating views on tablet %v", ti.Alias) if err := runSqlCommands(vscw.wr, ti, createViewCmds, abort); err != nil { processError("createViewCmds failed: %v", err) return } } for j := 0; j < vscw.destinationWriterCount; j++ { destinationWaitGroup.Add(1) go func() { defer destinationWaitGroup.Done() if err := executeFetchLoop(vscw.wr, ti, insertChannel, abort); err != nil { processError("executeFetchLoop failed: %v", err) } }() } }(vscw.destinationTablets[tabletAlias], insertChannels[i]) } // Now for each table, read data chunks and send them to all // insertChannels sourceWaitGroup := sync.WaitGroup{} sema := sync2.NewSemaphore(vscw.sourceReaderCount, 0) for tableIndex, td := range sourceSchemaDefinition.TableDefinitions { if td.Type == myproto.TABLE_VIEW { vscw.tableStatus[tableIndex].setState("view created") continue } vscw.tableStatus[tableIndex].setState("before copy") chunks, err := findChunks(vscw.wr, vscw.sourceTablet, td, vscw.minTableSizeForSplit, vscw.sourceReaderCount) if err != nil { return err } for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ { sourceWaitGroup.Add(1) go func(td *myproto.TableDefinition, tableIndex, chunkIndex int) { defer sourceWaitGroup.Done() sema.Acquire() defer sema.Release() vscw.tableStatus[tableIndex].setState("started the copy") // build the query, and start the streaming selectSQL := buildSQLFromChunks(vscw.wr, td, chunks, chunkIndex, vscw.sourceAlias.String()) qrr, err := NewQueryResultReaderForTablet(vscw.wr.TopoServer(), vscw.sourceAlias, selectSQL) if err != nil { processError("NewQueryResultReaderForTablet failed: %v", err) return } // process the data if err := vscw.processData(td, tableIndex, qrr, insertChannels, abort); err != nil { processError("QueryResultReader failed: %v", err) } }(td, tableIndex, chunkIndex) } } sourceWaitGroup.Wait() for _, c := range insertChannels { close(c) } destinationWaitGroup.Wait() if firstError != nil { return firstError } // do the post-copy alters if any if len(alterTablesCmds) > 0 { for _, tabletAlias := range vscw.destinationAliases { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() vscw.wr.Logger().Infof("Altering tables on tablet %v", ti.Alias) if err := runSqlCommands(vscw.wr, ti, alterTablesCmds, abort); err != nil { processError("alterTablesCmds failed on tablet %v: %v", ti.Alias, err) } }(vscw.destinationTablets[tabletAlias]) } destinationWaitGroup.Wait() if firstError != nil { return firstError } } // then create and populate the blp_checkpoint table if strings.Index(vscw.strategy, "populateBlpCheckpoint") != -1 { // get the current position from the source status, err := vscw.wr.TabletManagerClient().SlaveStatus(vscw.sourceTablet, 30*time.Second) if err != nil { return err } queries := make([]string, 0, 4) queries = append(queries, binlogplayer.CreateBlpCheckpoint()...) flags := "" if strings.Index(vscw.strategy, "dontStartBinlogPlayer") != -1 { flags = binlogplayer.BLP_FLAG_DONT_START } queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, status.Position, time.Now().Unix(), flags)) for _, tabletAlias := range vscw.destinationAliases { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() vscw.wr.Logger().Infof("Making and populating blp_checkpoint table on tablet %v", ti.Alias) if err := runSqlCommands(vscw.wr, ti, queries, abort); err != nil { processError("blp_checkpoint queries failed on tablet %v: %v", ti.Alias, err) } }(vscw.destinationTablets[tabletAlias]) } destinationWaitGroup.Wait() if firstError != nil { return firstError } } // Now we're done with data copy, update the shard's source info. if strings.Index(vscw.strategy, "skipSetSourceShards") != -1 { vscw.wr.Logger().Infof("Skipping setting SourceShard on destination shard.") } else { vscw.wr.Logger().Infof("Setting SourceShard on shard %v/%v", vscw.destinationKeyspace, vscw.destinationShard) if err := vscw.wr.SetSourceShards(vscw.destinationKeyspace, vscw.destinationShard, []topo.TabletAlias{vscw.sourceAlias}, vscw.tables); err != nil { return fmt.Errorf("Failed to set source shards: %v", err) } } // And force a schema reload on all destination tablets. // The master tablet will end up starting filtered replication // at this point. for _, tabletAlias := range vscw.destinationAliases { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() vscw.wr.Logger().Infof("Reloading schema on tablet %v", ti.Alias) if err := vscw.wr.TabletManagerClient().ReloadSchema(ti, 30*time.Second); err != nil { processError("ReloadSchema failed on tablet %v: %v", ti.Alias, err) } }(vscw.destinationTablets[tabletAlias]) } destinationWaitGroup.Wait() return firstError }
// copy phase: // - get schema on the source, filter tables // - create tables on all destinations // - copy the data func (vscw *VerticalSplitCloneWorker) copy() error { vscw.setState(stateVSCCopy) // get source schema sourceSchemaDefinition, err := vscw.wr.GetSchema(vscw.sourceAlias, vscw.tables, nil, true) if err != nil { return fmt.Errorf("cannot get schema from source %v: %v", vscw.sourceAlias, err) } if len(sourceSchemaDefinition.TableDefinitions) == 0 { return fmt.Errorf("no tables matching the table filter") } log.Infof("Source tablet has %v tables to copy", len(sourceSchemaDefinition.TableDefinitions)) vscw.mu.Lock() vscw.tableStatus = make([]tableStatus, len(sourceSchemaDefinition.TableDefinitions)) for i, td := range sourceSchemaDefinition.TableDefinitions { vscw.tableStatus[i].name = td.Name vscw.tableStatus[i].rowCount = td.RowCount } vscw.startTime = time.Now() vscw.mu.Unlock() // Create all the commands to create the destination schema: // - createDbCmds will create the database and the tables // - createViewCmds will create the views // - alterTablesCmds will modify the tables at the end if needed // (all need template substitution for {{.DatabaseName}}) createDbCmds := make([]string, 0, len(sourceSchemaDefinition.TableDefinitions)+1) createDbCmds = append(createDbCmds, sourceSchemaDefinition.DatabaseSchema) createViewCmds := make([]string, 0, 16) alterTablesCmds := make([]string, 0, 16) for i, td := range sourceSchemaDefinition.TableDefinitions { vscw.tableStatus[i].mu.Lock() if td.Type == myproto.TABLE_BASE_TABLE { create, alter, err := mysqlctl.MakeSplitCreateTableSql(td.Schema, "{{.DatabaseName}}", td.Name, vscw.strategy) if err != nil { return fmt.Errorf("MakeSplitCreateTableSql(%v) returned: %v", td.Name, err) } createDbCmds = append(createDbCmds, create) if alter != "" { alterTablesCmds = append(alterTablesCmds, alter) } vscw.tableStatus[i].state = "before table creation" vscw.tableStatus[i].rowCount = td.RowCount } else { createViewCmds = append(createViewCmds, td.Schema) vscw.tableStatus[i].state = "before view creation" vscw.tableStatus[i].rowCount = 0 } vscw.tableStatus[i].mu.Unlock() } // For each destination tablet (in parallel): // - create the schema // - setup the channels to send SQL data chunks // // mu protects the abort channel for closing, and firstError mu := sync.Mutex{} abort := make(chan struct{}) var firstError error processError := func(format string, args ...interface{}) { log.Errorf(format, args...) mu.Lock() if abort != nil { close(abort) abort = nil firstError = fmt.Errorf(format, args...) } mu.Unlock() } insertChannels := make([]chan string, len(vscw.destinationAliases)) destinationWaitGroup := sync.WaitGroup{} for i, tabletAlias := range vscw.destinationAliases { // we create one channel per destination tablet. It // is sized to have a buffer of a maximum of // destinationWriterCount * 2 items, to hopefully // always have data. We then have // destinationWriterCount go routines reading from it. insertChannels[i] = make(chan string, vscw.destinationWriterCount*2) destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo, insertChannel chan string) { defer destinationWaitGroup.Done() log.Infof("Creating tables on tablet %v", ti.Alias) if err := vscw.runSqlCommands(ti, createDbCmds, abort); err != nil { processError("createDbCmds failed: %v", err) return } if len(createViewCmds) > 0 { log.Infof("Creating views on tablet %v", ti.Alias) if err := vscw.runSqlCommands(ti, createViewCmds, abort); err != nil { processError("createViewCmds failed: %v", err) return } } for j := 0; j < vscw.destinationWriterCount; j++ { destinationWaitGroup.Add(1) go func() { defer destinationWaitGroup.Done() for { select { case cmd, ok := <-insertChannel: if !ok { return } cmd = "INSERT INTO `" + ti.DbName() + "`." + cmd _, err := vscw.wr.ActionInitiator().ExecuteFetch(ti, cmd, 0, false, true, 30*time.Second) if err != nil { processError("ExecuteFetch failed: %v", err) return } case <-abort: return } } }() } }(vscw.destinationTablets[tabletAlias], insertChannels[i]) } // Now for each table, read data chunks and send them to all // insertChannels sourceWaitGroup := sync.WaitGroup{} sema := sync2.NewSemaphore(vscw.sourceReaderCount, 0) for tableIndex, td := range sourceSchemaDefinition.TableDefinitions { if td.Type == myproto.TABLE_VIEW { vscw.tableStatus[tableIndex].setState("view created") continue } vscw.tableStatus[tableIndex].setState("before copy") chunks, err := vscw.findChunks(vscw.sourceTablet, td) if err != nil { return err } for chunkIndex := 0; chunkIndex < len(chunks)-1; chunkIndex++ { sourceWaitGroup.Add(1) go func(td myproto.TableDefinition, tableIndex, chunkIndex int) { defer sourceWaitGroup.Done() sema.Acquire() defer sema.Release() vscw.tableStatus[tableIndex].setState("started the copy") // build the query, and start the streaming selectSQL := "SELECT " + strings.Join(td.Columns, ", ") + " FROM " + td.Name if chunks[chunkIndex] != "" || chunks[chunkIndex+1] != "" { log.Infof("Starting to stream all data from table %v between '%v' and '%v'", td.Name, chunks[chunkIndex], chunks[chunkIndex+1]) clauses := make([]string, 0, 2) if chunks[chunkIndex] != "" { clauses = append(clauses, td.PrimaryKeyColumns[0]+">="+chunks[chunkIndex]) } if chunks[chunkIndex+1] != "" { clauses = append(clauses, td.PrimaryKeyColumns[0]+"<"+chunks[chunkIndex+1]) } selectSQL += " WHERE " + strings.Join(clauses, " AND ") } else { log.Infof("Starting to stream all data from table %v", td.Name) } if len(td.PrimaryKeyColumns) > 0 { selectSQL += " ORDER BY " + strings.Join(td.PrimaryKeyColumns, ", ") } qrr, err := NewQueryResultReaderForTablet(vscw.wr.TopoServer(), vscw.sourceAlias, selectSQL) if err != nil { processError("NewQueryResultReaderForTablet failed: %v", err) return } // process the data baseCmd := td.Name + "(" + strings.Join(td.Columns, ", ") + ") VALUES " loop: for { select { case r, ok := <-qrr.Output: if !ok { if err := qrr.Error(); err != nil { // error case processError("QueryResultReader failed: %v", err) return } // we're done with the data break loop } // send the rows to be inserted vscw.tableStatus[tableIndex].addCopiedRows(len(r.Rows)) cmd := baseCmd + makeValueString(qrr.Fields, r) for _, c := range insertChannels { c <- cmd } case <-abort: return } } }(td, tableIndex, chunkIndex) } } sourceWaitGroup.Wait() for _, c := range insertChannels { close(c) } destinationWaitGroup.Wait() if firstError != nil { return firstError } // do the post-copy alters if any if len(alterTablesCmds) > 0 { for _, tabletAlias := range vscw.destinationAliases { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() log.Infof("Altering tables on tablet %v", ti.Alias) if err := vscw.runSqlCommands(ti, alterTablesCmds, abort); err != nil { processError("alterTablesCmds failed on tablet %v: %v", ti.Alias, err) } }(vscw.destinationTablets[tabletAlias]) } destinationWaitGroup.Wait() if firstError != nil { return firstError } } // then create and populate the blp_checkpoint table if strings.Index(vscw.strategy, "populateBlpCheckpoint") != -1 { // get the current position from the source pos, err := vscw.wr.ActionInitiator().SlavePosition(vscw.sourceTablet, 30*time.Second) if err != nil { return err } queries := make([]string, 0, 4) queries = append(queries, binlogplayer.CreateBlpCheckpoint()...) flags := "" if strings.Index(vscw.strategy, "dontStartBinlogPlayer") != -1 { flags = binlogplayer.BLP_FLAG_DONT_START } queries = append(queries, binlogplayer.PopulateBlpCheckpoint(0, pos.MasterLogGTIDField.Value, time.Now().Unix(), flags)) for _, tabletAlias := range vscw.destinationAliases { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() log.Infof("Making and populating blp_checkpoint table on tablet %v", ti.Alias) if err := vscw.runSqlCommands(ti, queries, abort); err != nil { processError("blp_checkpoint queries failed on tablet %v: %v", ti.Alias, err) } }(vscw.destinationTablets[tabletAlias]) } destinationWaitGroup.Wait() if firstError != nil { return firstError } } // Now we're done with data copy, update the shard's source info. log.Infof("Setting SourceShard on shard %v/%v", vscw.destinationKeyspace, vscw.destinationShard) if err := vscw.wr.SetSourceShards(vscw.destinationKeyspace, vscw.destinationShard, []topo.TabletAlias{vscw.sourceAlias}, vscw.tables); err != nil { return fmt.Errorf("Failed to set source shards: %v", err) } // And force a schema reload on all destination tablets. // The master tablet will end up starting filtered replication // at this point. for _, tabletAlias := range vscw.destinationAliases { destinationWaitGroup.Add(1) go func(ti *topo.TabletInfo) { defer destinationWaitGroup.Done() log.Infof("Reloading schema on tablet %v", ti.Alias) if err := vscw.wr.ActionInitiator().ReloadSchema(ti, 30*time.Second); err != nil { processError("ReloadSchema failed on tablet %v: %v", ti.Alias, err) } }(vscw.destinationTablets[tabletAlias]) } destinationWaitGroup.Wait() return firstError }