func commandRestoreFromBackup(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { if err := subFlags.Parse(args); err != nil { return err } if subFlags.NArg() != 1 { return fmt.Errorf("The RestoreFromBackup command requires the <tablet alias> argument.") } tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias) if err != nil { return err } stream, err := wr.TabletManagerClient().RestoreFromBackup(ctx, tabletInfo.Tablet) if err != nil { return err } for { e, err := stream.Recv() switch err { case nil: logutil.LogEvent(wr.Logger(), e) case io.EOF: return nil default: return err } } }
func commandDemoteMaster(wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { if err := subFlags.Parse(args); err != nil { return err } if subFlags.NArg() != 1 { return fmt.Errorf("action DemoteMaster requires <tablet alias|zk tablet path>") } tabletAlias, err := tabletParamToTabletAlias(subFlags.Arg(0)) if err != nil { return err } tabletInfo, err := wr.TopoServer().GetTablet(tabletAlias) if err != nil { return err } return wr.TabletManagerClient().DemoteMaster(tabletInfo, wr.ActionTimeout()) }
func commandDemoteMaster(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { if err := subFlags.Parse(args); err != nil { return err } if subFlags.NArg() != 1 { return fmt.Errorf("action DemoteMaster requires <tablet alias>") } tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(0)) if err != nil { return err } tabletInfo, err := wr.TopoServer().GetTablet(ctx, tabletAlias) if err != nil { return err } _, err = wr.TabletManagerClient().DemoteMaster(ctx, tabletInfo.Tablet) return err }
// executeFetchLoop loops over the provided insertChannel // and sends the commands to the provided tablet. func executeFetchLoop(wr *wrangler.Wrangler, ti *topo.TabletInfo, insertChannel chan string, abort chan struct{}) error { for { select { case cmd, ok := <-insertChannel: if !ok { // no more to read, we're done return nil } cmd = "INSERT INTO `" + ti.DbName() + "`." + cmd _, err := wr.TabletManagerClient().ExecuteFetch(ti, cmd, 0, false, true, 30*time.Second) if err != nil { return fmt.Errorf("ExecuteFetch failed: %v", err) } case <-abort: // FIXME(alainjobart): note this select case // could be starved here, and we might miss // the abort in some corner cases. return nil } } }
// runSqlCommands will send the sql commands to the remote tablet. func runSqlCommands(wr *wrangler.Wrangler, ti *topo.TabletInfo, commands []string, abort chan struct{}) error { for _, command := range commands { command, err := fillStringTemplate(command, map[string]string{"DatabaseName": ti.DbName()}) if err != nil { return fmt.Errorf("fillStringTemplate failed: %v", err) } _, err = wr.TabletManagerClient().ExecuteFetch(ti, command, 0, false, true, 30*time.Second) if err != nil { return err } // check on abort select { case <-abort: return nil default: break } } return nil }
// FindChunks returns an array of chunks to use for splitting up a table // into multiple data chunks. It only works for tables with a primary key // (and the primary key first column is an integer type). // The array will always look like: // "", "value1", "value2", "" // A non-split tablet will just return: // "", "" func FindChunks(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, td *myproto.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) { result := []string{"", ""} // eliminate a few cases we don't split tables for if len(td.PrimaryKeyColumns) == 0 { // no primary key, what can we do? return result, nil } if td.DataLength < minTableSizeForSplit { // table is too small to split up return result, nil } // get the min and max of the leading column of the primary key query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name) shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, ti, query, 1, true) cancel() if err != nil { return nil, fmt.Errorf("ExecuteFetchAsApp: %v", err) } if len(qr.Rows) != 1 { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name) return result, nil } if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() { wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1]) return result, nil } switch qr.Fields[0].Type { case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24: minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw()) maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw()) if (qr.Fields[0].Flags & mproto.VT_UNSIGNED_FLAG) == 0 { // signed values, use int64 min, err := minNumeric.ParseInt64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err) return result, nil } max, err := maxNumeric.ParseInt64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err) return result, nil } interval := (max - min) / int64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := int64(1); i < int64(sourceReaderCount); i++ { result[i] = fmt.Sprintf("%v", min+interval*i) } return result, nil } // unsigned values, use uint64 min, err := minNumeric.ParseUint64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err) return result, nil } max, err := maxNumeric.ParseUint64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err) return result, nil } interval := (max - min) / uint64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := uint64(1); i < uint64(sourceReaderCount); i++ { result[i] = fmt.Sprintf("%v", min+interval*i) } return result, nil case mproto.VT_FLOAT, mproto.VT_DOUBLE: min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err) return result, nil } max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err) return result, nil } interval := (max - min) / float64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := 1; i < sourceReaderCount; i++ { result[i] = fmt.Sprintf("%v", min+interval*float64(i)) } return result, nil } wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name) return result, nil }
// executeFetchWithRetries will attempt to run ExecuteFetch for a single command, with a reasonably small timeout. // If will keep retrying the ExecuteFetch (for a finite but longer duration) if it fails due to a timeout or a // retriable application error. // // executeFetchWithRetries will also re-resolve the topology after errors, to be resistant to a reparent. // It takes in a tablet record that it will initially attempt to write to, and will return the final tablet // record that it used. func executeFetchWithRetries(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, r Resolver, shard string, command string) (*topo.TabletInfo, error) { retryDuration := 2 * time.Hour // We should keep retrying up until the retryCtx runs out retryCtx, retryCancel := context.WithTimeout(ctx, retryDuration) defer retryCancel() // Is this current attempt a retry of a previous attempt? isRetry := false for { tryCtx, cancel := context.WithTimeout(retryCtx, 2*time.Minute) _, err := wr.TabletManagerClient().ExecuteFetchAsApp(tryCtx, ti, command, 0, false) cancel() if err == nil { // success! return ti, nil } // If the ExecuteFetch call failed because of an application error, we will try to figure out why. // We need to extract the MySQL error number, and will attempt to retry if we think the error is recoverable. match := errExtract.FindStringSubmatch(err.Error()) var errNo string if len(match) == 2 { errNo = match[1] } switch { case wr.TabletManagerClient().IsTimeoutError(err): wr.Logger().Warningf("ExecuteFetch failed on %v; will retry because it was a timeout error: %v", ti, err) statsRetryCounters.Add("TimeoutError", 1) case errNo == "1290": wr.Logger().Warningf("ExecuteFetch failed on %v; will reresolve and retry because it's due to a MySQL read-only error: %v", ti, err) statsRetryCounters.Add("ReadOnly", 1) case errNo == "2002" || errNo == "2006": wr.Logger().Warningf("ExecuteFetch failed on %v; will reresolve and retry because it's due to a MySQL connection error: %v", ti, err) statsRetryCounters.Add("ConnectionError", 1) case errNo == "1062": if !isRetry { return ti, fmt.Errorf("ExecuteFetch failed on %v on the first attempt; not retrying as this is not a recoverable error: %v", ti, err) } wr.Logger().Infof("ExecuteFetch failed on %v with a duplicate entry error; marking this as a success, because of the likelihood that this query has already succeeded before being retried: %v", ti, err) return ti, nil default: // Unknown error return ti, err } t := time.NewTimer(*executeFetchRetryTime) // don't leak memory if the timer isn't triggered defer t.Stop() select { case <-retryCtx.Done(): if retryCtx.Err() == context.DeadlineExceeded { return ti, fmt.Errorf("failed to connect to destination tablet %v after retrying for %v", ti, retryDuration) } return ti, fmt.Errorf("interrupted while trying to run %v on tablet %v", command, ti) case <-t.C: // Re-resolve and retry 30s after the failure err = r.ResolveDestinationMasters(ctx) if err != nil { return ti, fmt.Errorf("unable to re-resolve masters for ExecuteFetch, due to: %v", err) } ti, err = r.GetDestinationMaster(shard) if err != nil { // At this point, we probably don't have a valid tablet record to return return nil, fmt.Errorf("unable to run ExecuteFetch due to: %v", err) } } isRetry = true } }
// generateChunks returns an array of chunks to use for splitting up a table // into multiple data chunks. It only works for tables with a primary key // whose first column is a numeric type. func generateChunks(ctx context.Context, wr *wrangler.Wrangler, tablet *topodatapb.Tablet, td *tabletmanagerdatapb.TableDefinition, chunkCount, minRowsPerChunk int) ([]chunk, error) { if len(td.PrimaryKeyColumns) == 0 { // No explicit primary key. Cannot chunk the rows then. wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks because it has no primary key columns. This will reduce the performance of the clone.", td.Name) return singleCompleteChunk, nil } if td.RowCount < 2*uint64(minRowsPerChunk) { // The automatic adjustment of "chunkCount" based on "minRowsPerChunk" // below would set "chunkCount" to less than 2 i.e. 1 or 0 chunks. // In practice in this case there should be exactly one chunk. // Return early in this case and notice the user about this. wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks because it has only %d rows.", td.Name, td.RowCount) return singleCompleteChunk, nil } if chunkCount == 1 { return singleCompleteChunk, nil } // Get the MIN and MAX of the leading column of the primary key. query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", escape(td.PrimaryKeyColumns[0]), escape(td.PrimaryKeyColumns[0]), escape(topoproto.TabletDbName(tablet)), escape(td.Name)) shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, tablet, true, []byte(query), 1) cancel() if err != nil { return nil, fmt.Errorf("Cannot determine MIN and MAX of the first primary key column. ExecuteFetchAsApp: %v", err) } if len(qr.Rows) != 1 { return nil, fmt.Errorf("Cannot determine MIN and MAX of the first primary key column. Zero rows were returned for the following query: %v", query) } result := sqltypes.Proto3ToResult(qr) min := result.Rows[0][0].ToNative() max := result.Rows[0][1].ToNative() if min == nil || max == nil { wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks, min or max is NULL: %v", td.Name, qr.Rows[0]) return singleCompleteChunk, nil } // Determine the average number of rows per chunk for the given chunkCount. avgRowsPerChunk := td.RowCount / uint64(chunkCount) if avgRowsPerChunk < uint64(minRowsPerChunk) { // Reduce the chunkCount to fulfill minRowsPerChunk. newChunkCount := td.RowCount / uint64(minRowsPerChunk) wr.Logger().Infof("table=%v: Reducing the number of chunks from the default %d to %d to make sure that each chunk has at least %d rows.", td.Name, chunkCount, newChunkCount, minRowsPerChunk) chunkCount = int(newChunkCount) } // TODO(mberlin): Write a unit test for this part of the function. var interval interface{} chunks := make([]chunk, chunkCount) switch min := min.(type) { case int64: max := max.(int64) interval = (max - min) / int64(chunkCount) if interval == 0 { wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks, interval=0: %v to %v", td.Name, min, max) return singleCompleteChunk, nil } case uint64: max := max.(uint64) interval = (max - min) / uint64(chunkCount) if interval == 0 { wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks, interval=0: %v to %v", td.Name, min, max) return singleCompleteChunk, nil } case float64: max := max.(float64) interval = (max - min) / float64(chunkCount) if interval == 0 { wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks, interval=0: %v to %v", td.Name, min, max) return singleCompleteChunk, nil } default: wr.Logger().Infof("table=%v: Not splitting the table into multiple chunks, primary key not numeric.", td.Name) return singleCompleteChunk, nil } // Create chunks. start := min for i := 0; i < chunkCount; i++ { end := add(start, interval) chunk, err := toChunk(start, end, i+1, chunkCount) if err != nil { return nil, err } chunks[i] = chunk start = end } // Clear out the MIN and MAX on the first and last chunk respectively // because other shards might have smaller or higher values than the one we // looked at. chunks[0].start = sqltypes.NULL chunks[chunkCount-1].end = sqltypes.NULL return chunks, nil }
// generateChunks returns an array of chunks to use for splitting up a table // into multiple data chunks. It only works for tables with a primary key // whose first column is a numeric type. func generateChunks(ctx context.Context, wr *wrangler.Wrangler, tablet *topodatapb.Tablet, td *tabletmanagerdatapb.TableDefinition, minTableSizeForSplit uint64, chunkCount int) ([]chunk, error) { if len(td.PrimaryKeyColumns) == 0 { // No explicit primary key. Cannot chunk the rows then. return singleCompleteChunk, nil } if td.DataLength < minTableSizeForSplit { // Table is too small to split up. return singleCompleteChunk, nil } if chunkCount == 1 { return singleCompleteChunk, nil } // Get the MIN and MAX of the leading column of the primary key. query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], topoproto.TabletDbName(tablet), td.Name) shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, tablet, true, []byte(query), 1) cancel() if err != nil { return nil, fmt.Errorf("Cannot determine MIN and MAX of the first primary key column. ExecuteFetchAsApp: %v", err) } if len(qr.Rows) != 1 { return nil, fmt.Errorf("Cannot determine MIN and MAX of the first primary key column. Zero rows were returned for the following query: %v", query) } result := sqltypes.Proto3ToResult(qr) min := result.Rows[0][0].ToNative() max := result.Rows[0][1].ToNative() if min == nil || max == nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v", td.Name, qr.Rows[0]) return singleCompleteChunk, nil } // TODO(mberlin): Write a unit test for this part of the function. chunks := make([]chunk, chunkCount) switch min := min.(type) { case int64: max := max.(int64) interval := (max - min) / int64(chunkCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v to %v", td.Name, min, max) return singleCompleteChunk, nil } start := min for i := 0; i < chunkCount; i++ { end := start + interval chunk, err := toChunk(start, end) if err != nil { return nil, err } chunks[i] = chunk start = end } case uint64: max := max.(uint64) interval := (max - min) / uint64(chunkCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v to %v", td.Name, min, max) return singleCompleteChunk, nil } start := min for i := 0; i < chunkCount; i++ { end := start + interval chunk, err := toChunk(start, end) if err != nil { return nil, err } chunks[i] = chunk start = end } case float64: max := max.(float64) interval := (max - min) / float64(chunkCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v to %v", td.Name, min, max) return singleCompleteChunk, nil } start := min for i := 0; i < chunkCount; i++ { end := start + interval chunk, err := toChunk(start, end) if err != nil { return nil, err } chunks[i] = chunk start = end } default: wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric.", td.Name) return singleCompleteChunk, nil } // Clear out the MIN and MAX on the first and last chunk respectively // because other shards might have smaller or higher values than the one we // looked at. chunks[0].start = sqltypes.NULL chunks[chunkCount-1].end = sqltypes.NULL return chunks, nil }
// FindChunks returns an array of chunks to use for splitting up a table // into multiple data chunks. It only works for tables with a primary key // (and the primary key first column is an integer type). // The array will always look like: // "", "value1", "value2", "" // A non-split tablet will just return: // "", "" func FindChunks(ctx context.Context, wr *wrangler.Wrangler, ti *topo.TabletInfo, td *tabletmanagerdatapb.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) { result := []string{"", ""} // eliminate a few cases we don't split tables for if len(td.PrimaryKeyColumns) == 0 { // no primary key, what can we do? return result, nil } if td.DataLength < minTableSizeForSplit { // table is too small to split up return result, nil } // get the min and max of the leading column of the primary key query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name) shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) qr, err := wr.TabletManagerClient().ExecuteFetchAsApp(shortCtx, ti, query, 1, true) cancel() if err != nil { return nil, fmt.Errorf("ExecuteFetchAsApp: %v", err) } if len(qr.Rows) != 1 { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name) return result, nil } // FIXME(alainjobart) this code is a bit clunky. I'd like to // convert the first row into an array of Values, and then // see which type they are and go from there. Can only happen after // Value has a full type. l0 := qr.Rows[0].Lengths[0] l1 := qr.Rows[0].Lengths[1] if l0 < 0 || l1 < 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v", td.Name, qr.Rows[0]) return result, nil } minValue := qr.Rows[0].Values[:l0] maxValue := qr.Rows[0].Values[l0 : l0+l1] switch { case sqltypes.IsSigned(qr.Fields[0].Type): minNumeric := sqltypes.MakeNumeric(minValue) maxNumeric := sqltypes.MakeNumeric(maxValue) min, err := minNumeric.ParseInt64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err) return result, nil } max, err := maxNumeric.ParseInt64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err) return result, nil } interval := (max - min) / int64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := int64(1); i < int64(sourceReaderCount); i++ { result[i] = fmt.Sprintf("%v", min+interval*i) } return result, nil case sqltypes.IsUnsigned(qr.Fields[0].Type): minNumeric := sqltypes.MakeNumeric(minValue) maxNumeric := sqltypes.MakeNumeric(maxValue) min, err := minNumeric.ParseUint64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err) return result, nil } max, err := maxNumeric.ParseUint64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err) return result, nil } interval := (max - min) / uint64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := uint64(1); i < uint64(sourceReaderCount); i++ { result[i] = fmt.Sprintf("%v", min+interval*i) } return result, nil case sqltypes.IsFloat(qr.Fields[0].Type): min, err := strconv.ParseFloat(string(minValue), 64) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, string(minValue), err) return result, nil } max, err := strconv.ParseFloat(string(maxValue), 64) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, string(maxValue), err) return result, nil } interval := (max - min) / float64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := 1; i < sourceReaderCount; i++ { result[i] = fmt.Sprintf("%v", min+interval*float64(i)) } return result, nil } wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name) return result, nil }