// NewSplitCloneWorker returns a new SplitCloneWorker object.
func NewSplitCloneWorker(wr *wrangler.Wrangler, cell, keyspace, shard string, excludeTables []string, strategyStr string, sourceReaderCount, destinationPackCount int, minTableSizeForSplit uint64, destinationWriterCount int) (Worker, error) {
strategy, err := mysqlctl.NewSplitStrategy(wr.Logger(), strategyStr)
if err != nil {
return nil, err
}
return &SplitCloneWorker{
wr: wr,
cell: cell,
keyspace: keyspace,
shard: shard,
excludeTables: excludeTables,
strategy: strategy,
sourceReaderCount: sourceReaderCount,
destinationPackCount: destinationPackCount,
minTableSizeForSplit: minTableSizeForSplit,
destinationWriterCount: destinationWriterCount,
cleaner: &wrangler.Cleaner{},
state: stateSCNotSarted,
ev: &events.SplitClone{
Cell: cell,
Keyspace: keyspace,
Shard: shard,
ExcludeTables: excludeTables,
Strategy: strategy.String(),
},
}, nil
}
// buildSQLFromChunks returns the SQL command to run to insert the data
// using the chunks definitions into the provided table.
func buildSQLFromChunks(wr *wrangler.Wrangler, td *myproto.TableDefinition, chunks []string, chunkIndex int, source string) string {
selectSQL := "SELECT " + strings.Join(td.Columns, ", ") + " FROM " + td.Name
if chunks[chunkIndex] != "" || chunks[chunkIndex+1] != "" {
wr.Logger().Infof("Starting to stream all data from tablet %v table %v between '%v' and '%v'", source, td.Name, chunks[chunkIndex], chunks[chunkIndex+1])
clauses := make([]string, 0, 2)
if chunks[chunkIndex] != "" {
clauses = append(clauses, td.PrimaryKeyColumns[0]+">="+chunks[chunkIndex])
}
if chunks[chunkIndex+1] != "" {
clauses = append(clauses, td.PrimaryKeyColumns[0]+"<"+chunks[chunkIndex+1])
}
selectSQL += " WHERE " + strings.Join(clauses, " AND ")
} else {
wr.Logger().Infof("Starting to stream all data from tablet %v table %v", source, td.Name)
}
if len(td.PrimaryKeyColumns) > 0 {
selectSQL += " ORDER BY " + strings.Join(td.PrimaryKeyColumns, ", ")
}
return selectSQL
}
func commandPruneActionLogs(wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
keepCount := subFlags.Int("keep-count", 10, "count to keep")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() == 0 {
return fmt.Errorf("action PruneActionLogs requires <zk action log path> ...")
}
paths, err := zkResolveWildcards(wr, subFlags.Args())
if err != nil {
return err
}
zkts, ok := wr.TopoServer().(*zktopo.Server)
if !ok {
return fmt.Errorf("PruneActionLogs requires a zktopo.Server")
}
var errCount sync2.AtomicInt32
wg := sync.WaitGroup{}
for _, zkActionLogPath := range paths {
wg.Add(1)
go func(zkActionLogPath string) {
defer wg.Done()
purgedCount, err := zkts.PruneActionLogs(zkActionLogPath, *keepCount)
if err == nil {
wr.Logger().Infof("%v pruned %v", zkActionLogPath, purgedCount)
} else {
wr.Logger().Errorf("%v pruning failed: %v", zkActionLogPath, err)
errCount.Add(1)
}
}(zkActionLogPath)
}
wg.Wait()
if errCount.Get() > 0 {
return fmt.Errorf("some errors occurred, check the log")
}
return nil
}
// findChecker:
// - find a rdonly instance in the keyspace / shard
// - mark it as checker
// - tag it with our worker process
func findChecker(wr *wrangler.Wrangler, cleaner *wrangler.Cleaner, cell, keyspace, shard string) (topo.TabletAlias, error) {
tabletAlias, err := findHealthyRdonlyEndPoint(wr, cell, keyspace, shard)
if err != nil {
return topo.TabletAlias{}, err
}
// We add the tag before calling ChangeSlaveType, so the destination
// vttablet reloads the worker URL when it reloads the tablet.
ourURL := servenv.ListeningURL.String()
wr.Logger().Infof("Adding tag[worker]=%v to tablet %v", ourURL, tabletAlias)
if err := wr.TopoServer().UpdateTabletFields(tabletAlias, func(tablet *topo.Tablet) error {
if tablet.Tags == nil {
tablet.Tags = make(map[string]string)
}
tablet.Tags["worker"] = ourURL
return nil
}); err != nil {
return topo.TabletAlias{}, err
}
// we remove the tag *before* calling ChangeSlaveType back, so
// we need to record this tag change after the change slave
// type change in the cleaner.
defer wrangler.RecordTabletTagAction(cleaner, tabletAlias, "worker", "")
wr.Logger().Infof("Changing tablet %v to 'checker'", tabletAlias)
wr.ResetActionTimeout(30 * time.Second)
if err := wr.ChangeType(tabletAlias, topo.TYPE_CHECKER, false /*force*/); err != nil {
return topo.TabletAlias{}, err
}
// Record a clean-up action to take the tablet back to rdonly.
// We will alter this one later on and let the tablet go back to
// 'spare' if we have stopped replication for too long on it.
wrangler.RecordChangeSlaveTypeAction(cleaner, tabletAlias, topo.TYPE_RDONLY)
return tabletAlias, nil
}
// findChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func findChunks(wr *wrangler.Wrangler, ti *topo.TabletInfo, td *myproto.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
result := []string{"", ""}
// eliminate a few cases we don't split tables for
if len(td.PrimaryKeyColumns) == 0 {
// no primary key, what can we do?
return result, nil
}
if td.DataLength < minTableSizeForSplit {
// table is too small to split up
return result, nil
}
// get the min and max of the leading column of the primary key
query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
qr, err := wr.TabletManagerClient().ExecuteFetch(ctx, ti, query, 1, true, false)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks: %v", td.Name, err)
return result, nil
}
cancel()
if len(qr.Rows) != 1 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
return result, nil
}
if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() {
wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1])
return result, nil
}
switch qr.Fields[0].Type {
case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24:
minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
if qr.Rows[0][0].Raw()[0] == '-' {
// signed values, use int64
min, err := minNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseInt64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / int64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := int64(1); i < int64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
}
// unsigned values, use uint64
min, err := minNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
return result, nil
}
max, err := maxNumeric.ParseUint64()
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
return result, nil
}
interval := (max - min) / uint64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := uint64(1); i < uint64(sourceReaderCount); i++ {
result[i] = fmt.Sprintf("%v", min+interval*i)
}
return result, nil
case mproto.VT_FLOAT, mproto.VT_DOUBLE:
min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err)
return result, nil
}
max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64)
if err != nil {
wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err)
return result, nil
}
interval := (max - min) / float64(sourceReaderCount)
if interval == 0 {
wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
return result, nil
}
result = make([]string, sourceReaderCount+1)
result[0] = ""
result[sourceReaderCount] = ""
for i := 1; i < sourceReaderCount; i++ {
result[i] = fmt.Sprintf("%v", min+interval*float64(i))
}
return result, nil
}
wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
return result, nil
}