// NewSplitCloneWorker returns a new SplitCloneWorker object.
func NewSplitCloneWorker(wr *wrangler.Wrangler, cell, keyspace, shard string, excludeTables []string, strategyStr string, sourceReaderCount, destinationPackCount int, minTableSizeForSplit uint64, destinationWriterCount int) (Worker, error) {
strategy, err := newSplitStrategy(wr.Logger(), strategyStr)
if err != nil {
return nil, err
}
return &SplitCloneWorker{
StatusWorker: NewStatusWorker(),
wr: wr,
cell: cell,
keyspace: keyspace,
shard: shard,
excludeTables: excludeTables,
strategy: strategy,
sourceReaderCount: sourceReaderCount,
destinationPackCount: destinationPackCount,
minTableSizeForSplit: minTableSizeForSplit,
destinationWriterCount: destinationWriterCount,
cleaner: &wrangler.Cleaner{},
ev: &events.SplitClone{
Cell: cell,
Keyspace: keyspace,
Shard: shard,
ExcludeTables: excludeTables,
Strategy: strategy.String(),
},
}, nil
}
func commandListBackups(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("action ListBackups requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return err
}
bucket := fmt.Sprintf("%v/%v", keyspace, shard)
bs, err := backupstorage.GetBackupStorage()
if err != nil {
return err
}
defer bs.Close()
bhs, err := bs.ListBackups(bucket)
if err != nil {
return err
}
for _, bh := range bhs {
wr.Logger().Printf("%v\n", bh.Name())
}
return nil
}
func commandRestoreFromBackup(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("The RestoreFromBackup command requires the <tablet alias> argument.")
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
stream, err := wr.TabletManagerClient().RestoreFromBackup(ctx, tabletInfo.Tablet)
if err != nil {
return err
}
for {
e, err := stream.Recv()
switch err {
case nil:
logutil.LogEvent(wr.Logger(), e)
case io.EOF:
return nil
default:
return err
}
}
}
func commandWorkflowCreate(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
if WorkflowManager == nil {
return fmt.Errorf("no workflow.Manager registered")
}
skipStart := subFlags.Bool("skip_start", false, "If set, the workflow will not be started.")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() < 1 {
return fmt.Errorf("the <factoryName> argument is required for the WorkflowCreate command")
}
factoryName := subFlags.Arg(0)
uuid, err := WorkflowManager.Create(ctx, factoryName, subFlags.Args()[1:])
if err != nil {
return err
}
wr.Logger().Printf("uuid: %v\n", uuid)
if !*skipStart {
return WorkflowManager.Start(ctx, uuid)
}
return nil
}
// NewVerticalSplitCloneWorker returns a new VerticalSplitCloneWorker object.
func NewVerticalSplitCloneWorker(wr *wrangler.Wrangler, cell, destinationKeyspace, destinationShard string, tables []string, strategyStr string, sourceReaderCount, destinationPackCount int, minTableSizeForSplit uint64, destinationWriterCount int) (Worker, error) {
strategy, err := mysqlctl.NewSplitStrategy(wr.Logger(), strategyStr)
if err != nil {
return nil, err
}
return &VerticalSplitCloneWorker{
StatusWorker: NewStatusWorker(),
wr: wr,
cell: cell,
destinationKeyspace: destinationKeyspace,
destinationShard: destinationShard,
tables: tables,
strategy: strategy,
sourceReaderCount: sourceReaderCount,
destinationPackCount: destinationPackCount,
minTableSizeForSplit: minTableSizeForSplit,
destinationWriterCount: destinationWriterCount,
cleaner: &wrangler.Cleaner{},
ev: &events.VerticalSplitClone{
Cell: cell,
Keyspace: destinationKeyspace,
Shard: destinationShard,
Tables: tables,
Strategy: strategy.String(),
},
}, nil
}
func commandVtGateSplitQuery(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
server := subFlags.String("server", "", "VtGate server to connect to")
bindVariables := newBindvars(subFlags)
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vtgate client")
splitCount := subFlags.Int("split_count", 16, "number of splits to generate")
keyspace := subFlags.String("keyspace", "", "keyspace to send query to")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <sql> argument is required for the VtGateSplitQuery command")
}
vtgateConn, err := vtgateconn.Dial(ctx, *server, *connectTimeout)
if err != nil {
return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err)
}
defer vtgateConn.Close()
r, err := vtgateConn.SplitQuery(ctx, *keyspace, tproto.BoundQuery{
Sql: subFlags.Arg(0),
BindVariables: *bindVariables,
}, *splitCount)
if err != nil {
return fmt.Errorf("SplitQuery failed: %v", err)
}
wr.Logger().Printf("%v\n", jscfg.ToJSON(r))
return nil
}
func commandVtGateExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
server := subFlags.String("server", "", "VtGate server to connect to")
bindVariables := newBindvars(subFlags)
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vtgate client")
tabletType := subFlags.String("tablet_type", "master", "tablet type to query")
json := subFlags.Bool("json", false, "Output JSON instead of human-readable table")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <sql> argument is required for the VtGateExecute command")
}
t, err := parseTabletType(*tabletType, []topodatapb.TabletType{topodatapb.TabletType_MASTER, topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY})
if err != nil {
return err
}
vtgateConn, err := vtgateconn.Dial(ctx, *server, *connectTimeout)
if err != nil {
return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err)
}
defer vtgateConn.Close()
qr, err := vtgateConn.Execute(ctx, subFlags.Arg(0), *bindVariables, t)
if err != nil {
return fmt.Errorf("Execute failed: %v", err)
}
if *json {
return printJSON(wr.Logger(), qr)
}
printQueryResult(loggerWriter{wr.Logger()}, qr)
return nil
}
func commandVtGateSplitQuery(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
server := subFlags.String("server", "", "VtGate server to connect to")
bindVariables := newBindvars(subFlags)
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vtgate client")
splitColumn := subFlags.String("split_column", "", "force the use of this column to split the query")
splitCount := subFlags.Int("split_count", 16, "number of splits to generate")
keyspace := subFlags.String("keyspace", "", "keyspace to send query to")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <sql> argument is required for the VtGateSplitQuery command")
}
vtgateConn, err := vtgateconn.Dial(ctx, *server, *connectTimeout, "")
if err != nil {
return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err)
}
defer vtgateConn.Close()
r, err := vtgateConn.SplitQuery(ctx, *keyspace, subFlags.Arg(0), *bindVariables, *splitColumn, int64(*splitCount))
if err != nil {
return fmt.Errorf("SplitQuery failed: %v", err)
}
return printJSON(wr.Logger(), r)
}
func commandVtTabletBegin(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <tablet_alias> argument is required for the VtTabletBegin command")
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
conn, err := tabletconn.GetDialer()(ctx, tabletInfo.Tablet, *connectTimeout)
if err != nil {
return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err)
}
defer conn.Close()
transactionID, err := conn.Begin(ctx)
if err != nil {
return fmt.Errorf("Begin failed: %v", err)
}
result := map[string]int64{
"transaction_id": transactionID,
}
return printJSON(wr.Logger(), result)
}
func getActions(wr *wrangler.Wrangler, zconn zk.Conn, actionPath string) ([]*actionnode.ActionNode, error) {
actions, _, err := zconn.Children(actionPath)
if err != nil {
return nil, fmt.Errorf("getActions failed: %v %v", actionPath, err)
}
sort.Strings(actions)
wg := sync.WaitGroup{}
mu := sync.Mutex{}
nodes := make([]*actionnode.ActionNode, 0, len(actions))
for _, action := range actions {
wg.Add(1)
go func(action string) {
defer wg.Done()
actionNodePath := path.Join(actionPath, action)
data, _, err := zconn.Get(actionNodePath)
if err != nil && !zookeeper.IsError(err, zookeeper.ZNONODE) {
wr.Logger().Warningf("getActions: %v %v", actionNodePath, err)
return
}
actionNode, err := actionnode.ActionNodeFromJson(data, actionNodePath)
if err != nil {
wr.Logger().Warningf("getActions: %v %v", actionNodePath, err)
return
}
mu.Lock()
nodes = append(nodes, actionNode)
mu.Unlock()
}(action)
}
wg.Wait()
return nodes, nil
}
func commandVtGateExecuteShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
server := subFlags.String("server", "", "VtGate server to connect to")
bindVariables := newBindvars(subFlags)
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vtgate client")
tabletType := subFlags.String("tablet_type", "master", "tablet type to query")
keyspace := subFlags.String("keyspace", "", "keyspace to send query to")
shardsStr := subFlags.String("shards", "", "comma-separated list of shards to send query to")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <sql> argument is required for the VtGateExecuteShard command")
}
t, err := parseTabletType(*tabletType, []topo.TabletType{topo.TYPE_MASTER, topo.TYPE_REPLICA, topo.TYPE_RDONLY})
if err != nil {
return err
}
var shards []string
if *shardsStr != "" {
shards = strings.Split(*shardsStr, ",")
}
vtgateConn, err := vtgateconn.Dial(ctx, *server, *connectTimeout)
if err != nil {
return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err)
}
defer vtgateConn.Close()
qr, err := vtgateConn.ExecuteShard(ctx, subFlags.Arg(0), *keyspace, shards, *bindVariables, t)
if err != nil {
return fmt.Errorf("Execute failed: %v", err)
}
wr.Logger().Printf("%v\n", jscfg.ToJSON(qr))
return nil
}
// FindHealthyRdonlyEndPoint returns a random healthy endpoint.
// Since we don't want to use them all, we require at least
// minHealthyEndPoints servers to be healthy.
// May block up to -wait_for_healthy_rdonly_endpoints_timeout.
func FindHealthyRdonlyEndPoint(ctx context.Context, wr *wrangler.Wrangler, cell, keyspace, shard string) (*topodatapb.TabletAlias, error) {
busywaitCtx, busywaitCancel := context.WithTimeout(ctx, *WaitForHealthyEndPointsTimeout)
defer busywaitCancel()
// create a discovery healthcheck, wait for it to have one rdonly
// endpoints at this point
healthCheck := discovery.NewHealthCheck(*remoteActionsTimeout, *healthcheckRetryDelay, *healthCheckTimeout, "" /* statsSuffix */)
watcher := discovery.NewShardReplicationWatcher(wr.TopoServer(), healthCheck, cell, keyspace, shard, *healthCheckTopologyRefresh, 5 /*topoReadConcurrency*/)
defer watcher.Stop()
defer healthCheck.Close()
if err := discovery.WaitForEndPoints(ctx, healthCheck, cell, keyspace, shard, []topodatapb.TabletType{topodatapb.TabletType_RDONLY}); err != nil {
return nil, fmt.Errorf("error waiting for rdonly endpoints for (%v,%v/%v): %v", cell, keyspace, shard, err)
}
var healthyEndpoints []*topodatapb.EndPoint
for {
select {
case <-busywaitCtx.Done():
return nil, fmt.Errorf("Not enough endpoints to choose from in (%v,%v/%v), have %v healthy ones, need at least %v Context Error: %v", cell, keyspace, shard, len(healthyEndpoints), *minHealthyEndPoints, busywaitCtx.Err())
default:
}
addrs := healthCheck.GetEndPointStatsFromTarget(keyspace, shard, topodatapb.TabletType_RDONLY)
healthyEndpoints = make([]*topodatapb.EndPoint, 0, len(addrs))
for _, addr := range addrs {
// Note we do not check the 'Serving' flag here.
// This is mainly to avoid the case where we run a
// Diff between a source and destination, and the source
// is not serving (disabled by TabletControl).
// When we switch the tablet to 'worker', it will
// go back to serving state.
if addr.Stats == nil || addr.Stats.HealthError != "" || addr.Stats.SecondsBehindMaster > 30 {
continue
}
healthyEndpoints = append(healthyEndpoints, addr.EndPoint)
}
if len(healthyEndpoints) >= *minHealthyEndPoints {
break
}
deadlineForLog, _ := busywaitCtx.Deadline()
wr.Logger().Infof("Waiting for enough endpoints to become available. available: %v required: %v Waiting up to %.1f more seconds.", len(healthyEndpoints), *minHealthyEndPoints, deadlineForLog.Sub(time.Now()).Seconds())
// Block for 1 second because 2 seconds is the -health_check_interval flag value in integration tests.
timer := time.NewTimer(1 * time.Second)
select {
case <-busywaitCtx.Done():
timer.Stop()
case <-timer.C:
}
}
// random server in the list is what we want
index := rand.Intn(len(healthyEndpoints))
return &topodatapb.TabletAlias{
Cell: cell,
Uid: healthyEndpoints[index].Uid,
}, nil
}
// newCloneWorker returns a new SplitCloneWorker object which is used both by
// the SplitClone and VerticalSplitClone command.
// TODO(mberlin): Rename SplitCloneWorker to cloneWorker.
func newCloneWorker(wr *wrangler.Wrangler, cloneType cloneType, cell, keyspace, shard string, online, offline bool, tables, excludeTables []string, strategyStr string, chunkCount, minRowsPerChunk, sourceReaderCount, writeQueryMaxRows, writeQueryMaxSize, writeQueryMaxRowsDelete, destinationWriterCount, minHealthyRdonlyTablets int, maxTPS, maxReplicationLag int64) (Worker, error) {
if cloneType != horizontalResharding && cloneType != verticalSplit {
return nil, fmt.Errorf("unknown cloneType: %v This is a bug. Please report", cloneType)
}
if tables != nil && len(tables) == 0 {
return nil, errors.New("list of tablets to be split out must not be empty")
}
strategy, err := newSplitStrategy(wr.Logger(), strategyStr)
if err != nil {
return nil, err
}
if maxTPS != throttler.MaxRateModuleDisabled {
wr.Logger().Infof("throttling enabled and set to a max of %v transactions/second", maxTPS)
}
if maxTPS != throttler.MaxRateModuleDisabled && maxTPS < int64(destinationWriterCount) {
return nil, fmt.Errorf("-max_tps must be >= -destination_writer_count: %v >= %v", maxTPS, destinationWriterCount)
}
if !online && !offline {
return nil, errors.New("at least one clone phase (-online, -offline) must be enabled (and not set to false)")
}
scw := &SplitCloneWorker{
StatusWorker: NewStatusWorker(),
wr: wr,
cloneType: cloneType,
cell: cell,
destinationKeyspace: keyspace,
shard: shard,
online: online,
offline: offline,
tables: tables,
excludeTables: excludeTables,
strategy: strategy,
chunkCount: chunkCount,
minRowsPerChunk: minRowsPerChunk,
sourceReaderCount: sourceReaderCount,
writeQueryMaxRows: writeQueryMaxRows,
writeQueryMaxSize: writeQueryMaxSize,
writeQueryMaxRowsDelete: writeQueryMaxRowsDelete,
destinationWriterCount: destinationWriterCount,
minHealthyRdonlyTablets: minHealthyRdonlyTablets,
maxTPS: maxTPS,
maxReplicationLag: maxReplicationLag,
cleaner: &wrangler.Cleaner{},
tabletTracker: NewTabletTracker(),
throttlers: make(map[string]*throttler.Throttler),
destinationDbNames: make(map[string]string),
tableStatusListOnline: &tableStatusList{},
tableStatusListOffline: &tableStatusList{},
}
scw.initializeEventDescriptor()
return scw, nil
}
func commandVtGateExecuteKeyspaceIds(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
server := subFlags.String("server", "", "VtGate server to connect to")
bindVariables := newBindvars(subFlags)
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vtgate client")
tabletType := subFlags.String("tablet_type", "master", "tablet type to query")
keyspace := subFlags.String("keyspace", "", "keyspace to send query to")
keyspaceIDsStr := subFlags.String("keyspace_ids", "", "comma-separated list of keyspace ids (in hex) that will map into shards to send query to")
options := subFlags.String("options", "", "execute options values as a text encoded proto of the ExecuteOptions structure")
json := subFlags.Bool("json", false, "Output JSON instead of human-readable table")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <sql> argument is required for the VtGateExecuteKeyspaceIds command")
}
t, err := parseTabletType(*tabletType, []topodatapb.TabletType{topodatapb.TabletType_MASTER, topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY})
if err != nil {
return err
}
var keyspaceIDs [][]byte
if *keyspaceIDsStr != "" {
keyspaceIDHexs := strings.Split(*keyspaceIDsStr, ",")
keyspaceIDs = make([][]byte, len(keyspaceIDHexs))
for i, keyspaceIDHex := range keyspaceIDHexs {
keyspaceIDs[i], err = hex.DecodeString(keyspaceIDHex)
if err != nil {
return fmt.Errorf("cannot hex-decode value %v '%v': %v", i, keyspaceIDHex, err)
}
}
}
executeOptions, err := parseExecuteOptions(*options)
if err != nil {
return err
}
vtgateConn, err := vtgateconn.Dial(ctx, *server, *connectTimeout, "")
if err != nil {
return fmt.Errorf("error connecting to vtgate '%v': %v", *server, err)
}
defer vtgateConn.Close()
qr, err := vtgateConn.ExecuteKeyspaceIds(ctx, subFlags.Arg(0), *keyspace, keyspaceIDs, *bindVariables, t, executeOptions)
if err != nil {
return fmt.Errorf("Execute failed: %v", err)
}
if *json {
return printJSON(wr.Logger(), qr)
}
printQueryResult(loggerWriter{wr.Logger()}, qr)
return nil
}
// FindHealthyRdonlyEndPoint returns a random healthy endpoint.
// Since we don't want to use them all, we require at least
// minHealthyEndPoints servers to be healthy.
// May block up to -wait_for_healthy_rdonly_endpoints_timeout.
func FindHealthyRdonlyEndPoint(ctx context.Context, wr *wrangler.Wrangler, cell, keyspace, shard string) (*topodatapb.TabletAlias, error) {
busywaitCtx, busywaitCancel := context.WithTimeout(ctx, *WaitForHealthyEndPointsTimeout)
defer busywaitCancel()
var healthyEndpoints []*topodatapb.EndPoint
for {
select {
case <-busywaitCtx.Done():
return nil, fmt.Errorf("Not enough endpoints to choose from in (%v,%v/%v), have %v healthy ones, need at least %v Context Error: %v", cell, keyspace, shard, len(healthyEndpoints), *minHealthyEndPoints, busywaitCtx.Err())
default:
}
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
endPoints, _, err := wr.TopoServer().GetEndPoints(shortCtx, cell, keyspace, shard, topodatapb.TabletType_RDONLY)
cancel()
if err != nil {
if err == topo.ErrNoNode {
// If the node doesn't exist, count that as 0 available rdonly instances.
endPoints = &topodatapb.EndPoints{}
} else {
return nil, fmt.Errorf("GetEndPoints(%v,%v,%v,rdonly) failed: %v", cell, keyspace, shard, err)
}
}
healthyEndpoints = make([]*topodatapb.EndPoint, 0, len(endPoints.Entries))
for _, entry := range endPoints.Entries {
if len(entry.HealthMap) == 0 {
healthyEndpoints = append(healthyEndpoints, entry)
}
}
if len(healthyEndpoints) < *minHealthyEndPoints {
deadlineForLog, _ := busywaitCtx.Deadline()
wr.Logger().Infof("Waiting for enough endpoints to become available. available: %v required: %v Waiting up to %.1f more seconds.", len(healthyEndpoints), *minHealthyEndPoints, deadlineForLog.Sub(time.Now()).Seconds())
// Block for 1 second because 2 seconds is the -health_check_interval flag value in integration tests.
timer := time.NewTimer(1 * time.Second)
select {
case <-busywaitCtx.Done():
timer.Stop()
case <-timer.C:
}
} else {
break
}
}
// random server in the list is what we want
index := rand.Intn(len(healthyEndpoints))
return &topodatapb.TabletAlias{
Cell: cell,
Uid: healthyEndpoints[index].Uid,
}, nil
}
// NewSplitCloneWorker returns a new SplitCloneWorker object.
func NewSplitCloneWorker(wr *wrangler.Wrangler, cell, keyspace, shard string, online, offline bool, excludeTables []string, strategyStr string, sourceReaderCount, writeQueryMaxRows, writeQueryMaxSize, writeQueryMaxRowsDelete int, minTableSizeForSplit uint64, destinationWriterCount, minHealthyRdonlyTablets int, maxTPS int64) (Worker, error) {
strategy, err := newSplitStrategy(wr.Logger(), strategyStr)
if err != nil {
return nil, err
}
if maxTPS != throttler.MaxRateModuleDisabled {
wr.Logger().Infof("throttling enabled and set to a max of %v transactions/second", maxTPS)
}
if maxTPS != throttler.MaxRateModuleDisabled && maxTPS < int64(destinationWriterCount) {
return nil, fmt.Errorf("-max_tps must be >= -destination_writer_count: %v >= %v", maxTPS, destinationWriterCount)
}
if !online && !offline {
return nil, errors.New("at least one clone phase (-online, -offline) must be enabled (and not set to false)")
}
return &SplitCloneWorker{
StatusWorker: NewStatusWorker(),
wr: wr,
cell: cell,
keyspace: keyspace,
shard: shard,
online: online,
offline: offline,
excludeTables: excludeTables,
strategy: strategy,
sourceReaderCount: sourceReaderCount,
writeQueryMaxRows: writeQueryMaxRows,
writeQueryMaxSize: writeQueryMaxSize,
writeQueryMaxRowsDelete: writeQueryMaxRowsDelete,
minTableSizeForSplit: minTableSizeForSplit,
destinationWriterCount: destinationWriterCount,
minHealthyRdonlyTablets: minHealthyRdonlyTablets,
maxTPS: maxTPS,
cleaner: &wrangler.Cleaner{},
tabletTracker: NewTabletTracker(),
destinationDbNames: make(map[string]string),
tableStatusListOnline: &tableStatusList{},
tableStatusListOffline: &tableStatusList{},
ev: &events.SplitClone{
Cell: cell,
Keyspace: keyspace,
Shard: shard,
ExcludeTables: excludeTables,
Strategy: strategy.String(),
},
}, nil
}
func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
transactionID := subFlags.Int("transaction_id", 0, "transaction id to use, if inside a transaction.")
bindVariables := newBindvars(subFlags)
keyspace := subFlags.String("keyspace", "", "keyspace the tablet belongs to")
shard := subFlags.String("shard", "", "shard the tablet belongs to")
tabletType := subFlags.String("tablet_type", "unknown", "tablet type we expect from the tablet (use unknown to use sessionId)")
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client")
json := subFlags.Bool("json", false, "Output JSON instead of human-readable table")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 2 {
return fmt.Errorf("the <tablet_alias> and <sql> arguments are required for the VtTabletExecute command")
}
tt, err := topoproto.ParseTabletType(*tabletType)
if err != nil {
return err
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
ep, err := topo.TabletEndPoint(tabletInfo.Tablet)
if err != nil {
return fmt.Errorf("cannot get EndPoint from tablet record: %v", err)
}
conn, err := tabletconn.GetDialer()(ctx, ep, *keyspace, *shard, tt, *connectTimeout)
if err != nil {
return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err)
}
defer conn.Close()
qr, err := conn.Execute(ctx, subFlags.Arg(1), *bindVariables, int64(*transactionID))
if err != nil {
return fmt.Errorf("Execute failed: %v", err)
}
if *json {
return printJSON(wr.Logger(), qr)
}
printQueryResult(loggerWriter{wr.Logger()}, qr)
return nil
}
func commandVtTabletUpdateStream(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
count := subFlags.Int("count", 1, "number of responses to wait for")
timestamp := subFlags.Int("timestamp", 0, "timestamp to start the stream from")
position := subFlags.String("position", "", "position to start the stream from")
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("The <tablet alias> argument is required for the VtTabletUpdateStream command.")
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
conn, err := tabletconn.GetDialer()(tabletInfo.Tablet, *connectTimeout)
if err != nil {
return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err)
}
stream, err := conn.UpdateStream(ctx, &querypb.Target{
Keyspace: tabletInfo.Tablet.Keyspace,
Shard: tabletInfo.Tablet.Shard,
TabletType: tabletInfo.Tablet.Type,
}, *position, int64(*timestamp))
if err != nil {
return err
}
for i := 0; i < *count; i++ {
se, err := stream.Recv()
if err != nil {
return fmt.Errorf("stream ended early: %v", err)
}
data, err := json.Marshal(se)
if err != nil {
wr.Logger().Errorf("cannot json-marshal structure: %v", err)
} else {
wr.Logger().Printf("%v\n", string(data))
}
}
return nil
}
func commandVtTabletStreamHealth(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
count := subFlags.Int("count", 1, "number of responses to wait for")
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("The <tablet alias> argument is required for the VtTabletStreamHealth command.")
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
ep, err := topo.TabletEndPoint(tabletInfo.Tablet)
if err != nil {
return fmt.Errorf("cannot get EndPoint from tablet record: %v", err)
}
// pass in a non-UNKNOWN tablet type to not use sessionId
conn, err := tabletconn.GetDialer()(ctx, ep, "", "", pb.TabletType_MASTER, *connectTimeout)
if err != nil {
return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err)
}
stream, errFunc, err := conn.StreamHealth(ctx)
if err != nil {
return err
}
for i := 0; i < *count; i++ {
shr, ok := <-stream
if !ok {
return fmt.Errorf("stream ended early: %v", errFunc())
}
data, err := json.Marshal(shr)
if err != nil {
wr.Logger().Errorf("cannot json-marshal structure: %v", err)
} else {
wr.Logger().Printf("%v\n", string(data))
}
}
return nil
}
// setAndStartWorker will set the current worker.
// We always log to both memory logger (for display on the web) and
// console logger (for records / display of command line worker).
func (wi *Instance) setAndStartWorker(wrk Worker, wr *wrangler.Wrangler) (chan struct{}, error) {
wi.currentWorkerMutex.Lock()
defer wi.currentWorkerMutex.Unlock()
if wi.currentWorker != nil {
return nil, fmt.Errorf("A worker is already in progress: %v", wi.currentWorker)
}
wi.currentWorker = wrk
wi.currentMemoryLogger = logutil.NewMemoryLogger()
wi.currentContext, wi.currentCancelFunc = context.WithCancel(wi.backgroundContext)
wi.lastRunError = nil
done := make(chan struct{})
wranglerLogger := wr.Logger()
if wr == wi.wr {
// If it's the default wrangler, do not reuse its logger because it may have been set before.
// Resuing it would result into an endless recursion.
wranglerLogger = logutil.NewConsoleLogger()
}
wr.SetLogger(logutil.NewTeeLogger(wi.currentMemoryLogger, wranglerLogger))
// one go function runs the worker, changes state when done
go func() {
log.Infof("Starting worker...")
var err error
// Catch all panics and always save the execution state at the end.
defer func() {
// The recovery code is a copy of servenv.HandlePanic().
if x := recover(); x != nil {
err = fmt.Errorf("uncaught %v panic: %v", "vtworker", x)
}
wi.currentWorkerMutex.Lock()
wi.currentContext = nil
wi.currentCancelFunc = nil
wi.lastRunError = err
wi.currentWorkerMutex.Unlock()
close(done)
}()
// run will take a long time
err = wrk.Run(wi.currentContext)
}()
return done, nil
}
func commandVtTabletExecute(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
transactionID := subFlags.Int("transaction_id", 0, "transaction id to use, if inside a transaction.")
bindVariables := newBindvars(subFlags)
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client")
options := subFlags.String("options", "", "execute options values as a text encoded proto of the ExecuteOptions structure")
json := subFlags.Bool("json", false, "Output JSON instead of human-readable table")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 2 {
return fmt.Errorf("the <tablet_alias> and <sql> arguments are required for the VtTabletExecute command")
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
executeOptions, err := parseExecuteOptions(*options)
if err != nil {
return err
}
conn, err := tabletconn.GetDialer()(tabletInfo.Tablet, *connectTimeout)
if err != nil {
return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err)
}
defer conn.Close(ctx)
qr, err := conn.Execute(ctx, &querypb.Target{
Keyspace: tabletInfo.Tablet.Keyspace,
Shard: tabletInfo.Tablet.Shard,
TabletType: tabletInfo.Tablet.Type,
}, subFlags.Arg(1), *bindVariables, int64(*transactionID), executeOptions)
if err != nil {
return fmt.Errorf("Execute failed: %v", err)
}
if *json {
return printJSON(wr.Logger(), qr)
}
printQueryResult(loggerWriter{wr.Logger()}, qr)
return nil
}
func commandWorker(wi *Instance, wr *wrangler.Wrangler, args []string, cell string, runFromCli bool) (Worker, error) {
action := args[0]
actionLowerCase := strings.ToLower(action)
for _, group := range commands {
for _, cmd := range group.Commands {
if strings.ToLower(cmd.Name) == actionLowerCase {
var subFlags *flag.FlagSet
if runFromCli {
subFlags = flag.NewFlagSet(action, flag.ExitOnError)
} else {
subFlags = flag.NewFlagSet(action, flag.ContinueOnError)
}
// The command may be run from an RPC and may not log to the console.
// The Wrangler logger defines where the output has to go.
subFlags.SetOutput(logutil.NewLoggerWriter(wr.Logger()))
subFlags.Usage = func() {
wr.Logger().Printf("Usage: %s %s %s\n\n", os.Args[0], cmd.Name, cmd.Params)
wr.Logger().Printf("%s\n\n", cmd.Help)
subFlags.PrintDefaults()
}
return cmd.Method(wi, wr, subFlags, args[1:])
}
}
}
if runFromCli {
flag.Usage()
} else {
PrintAllCommands(wr.Logger())
}
return nil, fmt.Errorf("unknown command: %v", action)
}
func waitForHealthyRdonlyTablets(ctx context.Context, wr *wrangler.Wrangler, tsc *discovery.TabletStatsCache, cell, keyspace, shard string, minHealthyRdonlyTablets int, timeout time.Duration) ([]discovery.TabletStats, error) {
busywaitCtx, busywaitCancel := context.WithTimeout(ctx, timeout)
defer busywaitCancel()
start := time.Now()
deadlineForLog, _ := busywaitCtx.Deadline()
log.V(2).Infof("Waiting for enough healthy RDONLY tablets to become available in (%v,%v/%v). required: %v Waiting up to %.1f seconds.",
cell, keyspace, shard, minHealthyRdonlyTablets, deadlineForLog.Sub(time.Now()).Seconds())
// Wait for at least one RDONLY tablet initially before checking the list.
if err := tsc.WaitForTablets(busywaitCtx, cell, keyspace, shard, []topodatapb.TabletType{topodatapb.TabletType_RDONLY}); err != nil {
return nil, fmt.Errorf("error waiting for RDONLY tablets for (%v,%v/%v): %v", cell, keyspace, shard, err)
}
var healthyTablets []discovery.TabletStats
for {
select {
case <-busywaitCtx.Done():
return nil, fmt.Errorf("not enough healthy RDONLY tablets to choose from in (%v,%v/%v), have %v healthy ones, need at least %v Context error: %v",
cell, keyspace, shard, len(healthyTablets), minHealthyRdonlyTablets, busywaitCtx.Err())
default:
}
healthyTablets = discovery.RemoveUnhealthyTablets(tsc.GetTabletStats(keyspace, shard, topodatapb.TabletType_RDONLY))
if len(healthyTablets) >= minHealthyRdonlyTablets {
break
}
deadlineForLog, _ := busywaitCtx.Deadline()
wr.Logger().Infof("Waiting for enough healthy RDONLY tablets to become available (%v,%v/%v). available: %v required: %v Waiting up to %.1f more seconds.",
cell, keyspace, shard, len(healthyTablets), minHealthyRdonlyTablets, deadlineForLog.Sub(time.Now()).Seconds())
// Block for 1 second because 2 seconds is the -health_check_interval flag value in integration tests.
timer := time.NewTimer(1 * time.Second)
select {
case <-busywaitCtx.Done():
timer.Stop()
case <-timer.C:
}
}
log.V(2).Infof("At least %v healthy RDONLY tablets are available in (%v,%v/%v) (required: %v). Took %.1f seconds to find this out.",
len(healthyTablets), cell, keyspace, shard, minHealthyRdonlyTablets, time.Now().Sub(start).Seconds())
return healthyTablets, nil
}
func commandWorkflowTree(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
if WorkflowManager == nil {
return fmt.Errorf("no workflow.Manager registered")
}
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 0 {
return fmt.Errorf("the WorkflowTree command takes no parameter")
}
tree, err := WorkflowManager.NodeManager().GetFullTree()
if err != nil {
return err
}
wr.Logger().Printf("%v\n", string(tree))
return nil
}
// Does a topo lookup for a single shard, and returns:
// 1. Slice of all tablet aliases for the shard.
// 2. Map of tablet alias : tablet record for all tablets.
func resolveRefreshTabletsForShard(ctx context.Context, keyspace, shard string, wr *wrangler.Wrangler) (refreshAliases []*topodatapb.TabletAlias, refreshTablets map[topodatapb.TabletAlias]*topo.TabletInfo, err error) {
// Keep a long timeout, because we really don't want the copying to succeed, and then the worker to fail at the end.
shortCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
refreshAliases, err = wr.TopoServer().FindAllTabletAliasesInShard(shortCtx, keyspace, shard)
cancel()
if err != nil {
return nil, nil, fmt.Errorf("cannot find all refresh target tablets in %v/%v: %v", keyspace, shard, err)
}
wr.Logger().Infof("Found %v refresh target aliases in shard %v/%v", len(refreshAliases), keyspace, shard)
shortCtx, cancel = context.WithTimeout(ctx, 5*time.Minute)
refreshTablets, err = wr.TopoServer().GetTabletMap(shortCtx, refreshAliases)
cancel()
if err != nil {
return nil, nil, fmt.Errorf("cannot read all refresh target tablets in %v/%v: %v",
keyspace, shard, err)
}
return refreshAliases, refreshTablets, nil
}
func commandVtTabletBegin(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
keyspace := subFlags.String("keyspace", "", "keyspace the tablet belongs to")
shard := subFlags.String("shard", "", "shard the tablet belongs to")
tabletType := subFlags.String("tablet_type", "unknown", "tablet type we expect from the tablet (use unknown to use sessionId)")
connectTimeout := subFlags.Duration("connect_timeout", 30*time.Second, "Connection timeout for vttablet client")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("the <tablet_alias> argument is required for the VtTabletBegin command")
}
tt, err := topoproto.ParseTabletType(*tabletType)
if err != nil {
return err
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0))
if err != nil {
return err
}
tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
if err != nil {
return err
}
ep, err := topo.TabletEndPoint(tabletInfo.Tablet)
if err != nil {
return fmt.Errorf("cannot get EndPoint from tablet record: %v", err)
}
conn, err := tabletconn.GetDialer()(ctx, ep, *keyspace, *shard, tt, *connectTimeout)
if err != nil {
return fmt.Errorf("cannot connect to tablet %v: %v", tabletAlias, err)
}
defer conn.Close()
transactionID, err := conn.Begin(ctx)
if err != nil {
return fmt.Errorf("Begin failed: %v", err)
}
result := map[string]int64{
"transaction_id": transactionID,
}
return printJSON(wr.Logger(), result)
}
// buildSQLFromChunks returns the SQL command to run to insert the data
// using the chunks definitions into the provided table.
func buildSQLFromChunks(wr *wrangler.Wrangler, td *myproto.TableDefinition, chunks []string, chunkIndex int, source string) string {
selectSQL := "SELECT " + strings.Join(td.Columns, ", ") + " FROM " + td.Name
if chunks[chunkIndex] != "" || chunks[chunkIndex+1] != "" {
wr.Logger().Infof("Starting to stream all data from tablet %v table %v between '%v' and '%v'", source, td.Name, chunks[chunkIndex], chunks[chunkIndex+1])
clauses := make([]string, 0, 2)
if chunks[chunkIndex] != "" {
clauses = append(clauses, td.PrimaryKeyColumns[0]+">="+chunks[chunkIndex])
}
if chunks[chunkIndex+1] != "" {
clauses = append(clauses, td.PrimaryKeyColumns[0]+"<"+chunks[chunkIndex+1])
}
selectSQL += " WHERE " + strings.Join(clauses, " AND ")
} else {
wr.Logger().Infof("Starting to stream all data from tablet %v table %v", source, td.Name)
}
if len(td.PrimaryKeyColumns) > 0 {
selectSQL += " ORDER BY " + strings.Join(td.PrimaryKeyColumns, ", ")
}
return selectSQL
}
// FindWorkerTablet will:
// - find a rdonly instance in the keyspace / shard
// - mark it as worker
// - tag it with our worker process
func FindWorkerTablet(ctx context.Context, wr *wrangler.Wrangler, cleaner *wrangler.Cleaner, cell, keyspace, shard string) (*topodatapb.TabletAlias, error) {
tabletAlias, err := FindHealthyRdonlyEndPoint(ctx, wr, cell, keyspace, shard)
if err != nil {
return nil, err
}
// We add the tag before calling ChangeSlaveType, so the destination
// vttablet reloads the worker URL when it reloads the tablet.
ourURL := servenv.ListeningURL.String()
wr.Logger().Infof("Adding tag[worker]=%v to tablet %v", ourURL, topoproto.TabletAliasString(tabletAlias))
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
_, err = wr.TopoServer().UpdateTabletFields(shortCtx, tabletAlias, func(tablet *topodatapb.Tablet) error {
if tablet.Tags == nil {
tablet.Tags = make(map[string]string)
}
tablet.Tags["worker"] = ourURL
return nil
})
cancel()
if err != nil {
return nil, err
}
// Using "defer" here because we remove the tag *before* calling
// ChangeSlaveType back, so we need to record this tag change after the change
// slave type change in the cleaner.
defer wrangler.RecordTabletTagAction(cleaner, tabletAlias, "worker", "")
wr.Logger().Infof("Changing tablet %v to '%v'", topoproto.TabletAliasString(tabletAlias), topodatapb.TabletType_WORKER)
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
err = wr.ChangeSlaveType(shortCtx, tabletAlias, topodatapb.TabletType_WORKER)
cancel()
if err != nil {
return nil, err
}
// Record a clean-up action to take the tablet back to rdonly.
// We will alter this one later on and let the tablet go back to
// 'spare' if we have stopped replication for too long on it.
wrangler.RecordChangeSlaveTypeAction(cleaner, tabletAlias, topodatapb.TabletType_RDONLY)
return tabletAlias, nil
}
// NewVerticalSplitCloneWorker returns a new VerticalSplitCloneWorker object.
func NewVerticalSplitCloneWorker(wr *wrangler.Wrangler, cell, destinationKeyspace, destinationShard string, tables []string, strategyStr string, sourceReaderCount, destinationPackCount int, minTableSizeForSplit uint64, destinationWriterCount, minHealthyRdonlyTablets int, maxTPS int64) (Worker, error) {
if len(tables) == 0 {
return nil, errors.New("list of tablets to be split out must not be empty")
}
strategy, err := newSplitStrategy(wr.Logger(), strategyStr)
if err != nil {
return nil, err
}
if maxTPS != throttler.MaxRateModuleDisabled {
wr.Logger().Infof("throttling enabled and set to a max of %v transactions/second", maxTPS)
}
if maxTPS != throttler.MaxRateModuleDisabled && maxTPS < int64(destinationWriterCount) {
return nil, fmt.Errorf("-max_tps must be >= -destination_writer_count: %v >= %v", maxTPS, destinationWriterCount)
}
return &VerticalSplitCloneWorker{
StatusWorker: NewStatusWorker(),
wr: wr,
cell: cell,
destinationKeyspace: destinationKeyspace,
destinationShard: destinationShard,
tables: tables,
strategy: strategy,
sourceReaderCount: sourceReaderCount,
destinationPackCount: destinationPackCount,
minTableSizeForSplit: minTableSizeForSplit,
destinationWriterCount: destinationWriterCount,
minHealthyRdonlyTablets: minHealthyRdonlyTablets,
maxTPS: maxTPS,
cleaner: &wrangler.Cleaner{},
destinationDbNames: make(map[string]string),
ev: &events.VerticalSplitClone{
Cell: cell,
Keyspace: destinationKeyspace,
Shard: destinationShard,
Tables: tables,
Strategy: strategy.String(),
},
}, nil
}
func commandPruneActionLogs(wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
keepCount := subFlags.Int("keep-count", 10, "count to keep")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() == 0 {
return fmt.Errorf("action PruneActionLogs requires <zk action log path> ...")
}
paths, err := resolveWildcards(wr, subFlags.Args())
if err != nil {
return err
}
zkts, ok := wr.TopoServer().(*zktopo.Server)
if !ok {
return fmt.Errorf("PruneActionLogs requires a zktopo.Server")
}
var errCount sync2.AtomicInt32
wg := sync.WaitGroup{}
for _, zkActionLogPath := range paths {
wg.Add(1)
go func(zkActionLogPath string) {
defer wg.Done()
purgedCount, err := zkts.PruneActionLogs(zkActionLogPath, *keepCount)
if err == nil {
wr.Logger().Infof("%v pruned %v", zkActionLogPath, purgedCount)
} else {
wr.Logger().Errorf("%v pruning failed: %v", zkActionLogPath, err)
errCount.Add(1)
}
}(zkActionLogPath)
}
wg.Wait()
if errCount.Get() > 0 {
return fmt.Errorf("some errors occurred, check the log")
}
return nil
}