// runSqlCommands will send the sql commands to the remote tablet. func runSqlCommands(wr *wrangler.Wrangler, ti *topo.TabletInfo, commands []string, abort chan struct{}, disableBinLogs bool) error { for _, command := range commands { command, err := fillStringTemplate(command, map[string]string{"DatabaseName": ti.DbName()}) if err != nil { return fmt.Errorf("fillStringTemplate failed: %v", err) } ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) _, err = wr.TabletManagerClient().ExecuteFetch(ctx, ti, command, 0, false, disableBinLogs) if err != nil { return err } cancel() // check on abort select { case <-abort: return nil default: break } } return nil }
// applySqlShard applies a given SQL change on a given tablet alias. It allows executing arbitrary // SQL statements, but doesn't return any results, so it's only useful for SQL statements // that would be run for their effects (e.g., CREATE). // It works by applying the SQL statement on the shard's master tablet with replication turned on. // Thus it should be used only for changes that can be applies on a live instance without causing issues; // it shouldn't be used for anything that will require a pivot. // The SQL statement string is expected to have {{.DatabaseName}} in place of the actual db name. func (wr *Wrangler) applySqlShard(tabletInfo *topo.TabletInfo, change string) error { filledChange, err := fillStringTemplate(change, map[string]string{"DatabaseName": tabletInfo.DbName()}) if err != nil { return fmt.Errorf("fillStringTemplate failed: %v", err) } ctx, cancel := context.WithTimeout(wr.ctx, 30*time.Second) defer cancel() // Need to make sure that we enable binlog, since we're only applying the statement on masters. _, err = wr.tmc.ExecuteFetch(ctx, tabletInfo, filledChange, 0, false, false) return err }
// executeFetchLoop loops over the provided insertChannel // and sends the commands to the provided tablet. func executeFetchLoop(wr *wrangler.Wrangler, ti *topo.TabletInfo, insertChannel chan string, abort chan struct{}, disableBinLogs bool) error { for { select { case cmd, ok := <-insertChannel: if !ok { // no more to read, we're done return nil } cmd = "INSERT INTO `" + ti.DbName() + "`." + cmd ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) _, err := wr.TabletManagerClient().ExecuteFetch(ctx, ti, cmd, 0, false, disableBinLogs) if err != nil { return fmt.Errorf("ExecuteFetch failed: %v", err) } cancel() case <-abort: // FIXME(alainjobart): note this select case // could be starved here, and we might miss // the abort in some corner cases. return nil } } }
// findChunks returns an array of chunks to use for splitting up a table // into multiple data chunks. It only works for tables with a primary key // (and the primary key first column is an integer type). // The array will always look like: // "", "value1", "value2", "" // A non-split tablet will just return: // "", "" func findChunks(wr *wrangler.Wrangler, ti *topo.TabletInfo, td *myproto.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) { result := []string{"", ""} // eliminate a few cases we don't split tables for if len(td.PrimaryKeyColumns) == 0 { // no primary key, what can we do? return result, nil } if td.DataLength < minTableSizeForSplit { // table is too small to split up return result, nil } // get the min and max of the leading column of the primary key query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name) ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) qr, err := wr.TabletManagerClient().ExecuteFetch(ctx, ti, query, 1, true, false) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks: %v", td.Name, err) return result, nil } cancel() if len(qr.Rows) != 1 { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name) return result, nil } if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() { wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1]) return result, nil } switch qr.Fields[0].Type { case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24: minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw()) maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw()) if qr.Rows[0][0].Raw()[0] == '-' { // signed values, use int64 min, err := minNumeric.ParseInt64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err) return result, nil } max, err := maxNumeric.ParseInt64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err) return result, nil } interval := (max - min) / int64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := int64(1); i < int64(sourceReaderCount); i++ { result[i] = fmt.Sprintf("%v", min+interval*i) } return result, nil } // unsigned values, use uint64 min, err := minNumeric.ParseUint64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err) return result, nil } max, err := maxNumeric.ParseUint64() if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err) return result, nil } interval := (max - min) / uint64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := uint64(1); i < uint64(sourceReaderCount); i++ { result[i] = fmt.Sprintf("%v", min+interval*i) } return result, nil case mproto.VT_FLOAT, mproto.VT_DOUBLE: min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err) return result, nil } max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64) if err != nil { wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err) return result, nil } interval := (max - min) / float64(sourceReaderCount) if interval == 0 { wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min) return result, nil } result = make([]string, sourceReaderCount+1) result[0] = "" result[sourceReaderCount] = "" for i := 1; i < sourceReaderCount; i++ { result[i] = fmt.Sprintf("%v", min+interval*float64(i)) } return result, nil } wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name) return result, nil }