func main() {
flag.Parse()
servenv.Init()
defer servenv.Close()
keyRange, err := key.ParseKeyRangeParts(*start, *end)
if err != nil {
log.Fatalf("Invalid key range: %v", err)
}
if *dbConfigFile == "" {
log.Fatalf("Cannot start without db-config-file")
}
dbConfig, err := readDbConfig(*dbConfigFile)
if err != nil {
log.Fatalf("Cannot read db config file: %v", err)
}
var t []string
if *tables != "" {
t = strings.Split(*tables, ",")
for i, table := range t {
t[i] = strings.TrimSpace(table)
}
}
interrupted := make(chan struct{})
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM)
go func() {
for _ = range c {
close(interrupted)
}
}()
var vtClient mysqlctl.VtClient
vtClient = mysqlctl.NewDbClient(dbConfig)
err = vtClient.Connect()
if err != nil {
log.Fatalf("error in initializing dbClient: %v", err)
}
brs, err := mysqlctl.ReadStartPosition(vtClient, uint32(*uid))
if err != nil {
log.Fatalf("Cannot read start position from db: %v", err)
}
if *debug {
vtClient = mysqlctl.NewDummyVtClient()
}
blp, err := mysqlctl.NewBinlogPlayer(vtClient, keyRange, uint32(*uid), brs, t, *txnBatch, time.Duration(*maxTxnInterval)*time.Second, *execDdl)
if err != nil {
log.Fatalf("error in initializing binlog player: %v", err)
}
err = blp.ApplyBinlogEvents(interrupted)
if err != nil {
log.Errorf("Error in applying binlog events, err %v", err)
}
log.Infof("vt_binlog_player done")
}
func TestVTGateStreamExecuteKeyRanges(t *testing.T) {
s := createSandbox("TestVTGateStreamExecuteKeyRanges")
sbc := &sandboxConn{}
s.MapTestConn("-20", sbc)
sbc1 := &sandboxConn{}
s.MapTestConn("20-40", sbc1)
kr, err := key.ParseKeyRangeParts("", "20")
// Test for successful execution
var qrs []*proto.QueryResult
err = rpcVTGate.StreamExecuteKeyRanges(context.Background(),
"query",
nil,
"TestVTGateStreamExecuteKeyRanges",
[]key.KeyRange{kr},
pb.TabletType_MASTER,
func(r *proto.QueryResult) error {
qrs = append(qrs, r)
return nil
})
if err != nil {
t.Errorf("want nil, got %v", err)
}
row := new(proto.QueryResult)
row.Result = singleRowResult
want := []*proto.QueryResult{row}
if !reflect.DeepEqual(want, qrs) {
t.Errorf("want \n%+v, got \n%+v", want, qrs)
}
// Test for successful execution - multiple shards
kr, err = key.ParseKeyRangeParts("10", "40")
err = rpcVTGate.StreamExecuteKeyRanges(context.Background(),
"query",
nil,
"TestVTGateStreamExecuteKeyRanges",
[]key.KeyRange{kr},
pb.TabletType_MASTER,
func(r *proto.QueryResult) error {
qrs = append(qrs, r)
return nil
})
if err != nil {
t.Errorf("want nil, got %v", err)
}
}
func multiRestoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
starts := subFlags.String("starts", "", "starts of the key range")
ends := subFlags.String("ends", "", "ends of the key range")
fetchRetryCount := subFlags.Int("fetch_retry_count", 3, "how many times to retry a failed transfer")
concurrency := subFlags.Int("concurrency", 8, "how many concurrent db inserts to run simultaneously")
fetchConcurrency := subFlags.Int("fetch_concurrency", 4, "how many files to fetch simultaneously")
insertTableConcurrency := subFlags.Int("insert_table_concurrency", 4, "how many myisam tables to load into a single destination table simultaneously")
strategy := subFlags.String("strategy", "", "which strategy to use for restore, can contain:\n"+
" skipAutoIncrement(TTT): we won't add the AUTO_INCREMENT back to that table\n"+
" delayPrimaryKey: we won't add the primary key until after the table is populated\n"+
" delaySecondaryIndexes: we won't add the secondary indexes until after the table is populated\n"+
" useMyIsam: create the table as MyISAM, then convert it to InnoDB after population\n"+
" writeBinLogs: write all operations to the binlogs")
subFlags.Parse(args)
if subFlags.NArg() < 2 {
log.Fatalf("multirestore requires <destination_dbname> <source_host>[/<source_dbname>]... %v", args)
}
startArray := strings.Split(*starts, ",")
endArray := strings.Split(*ends, ",")
if len(startArray) != len(endArray) || len(startArray) != subFlags.NArg()-1 {
log.Fatalf("Need as many starts and ends as source URLs")
}
keyRanges := make([]key.KeyRange, len(startArray))
for i, s := range startArray {
var err error
keyRanges[i], err = key.ParseKeyRangeParts(s, endArray[i])
if err != nil {
log.Fatalf("Invalid start or end: %v", err)
}
}
dbName, dbis := subFlags.Arg(0), subFlags.Args()[1:]
sources := make([]*url.URL, len(dbis))
for i, dbi := range dbis {
if !strings.HasPrefix(dbi, "vttp://") && !strings.HasPrefix(dbi, "http://") {
dbi = "vttp://" + dbi
}
dbUrl, err := url.Parse(dbi)
if err != nil {
log.Fatalf("incorrect source url: %v", err)
}
sources[i] = dbUrl
}
if err := mysqld.MultiRestore(dbName, keyRanges, sources, nil, *concurrency, *fetchConcurrency, *insertTableConcurrency, *fetchRetryCount, *strategy); err != nil {
log.Fatalf("multirestore failed: %v", err)
}
}
// ValidateShardName takes a shard name and sanitizes it, and also returns
// the KeyRange.
func ValidateShardName(shard string) (string, *topodatapb.KeyRange, error) {
if !IsShardUsingRangeBasedSharding(shard) {
return shard, nil, nil
}
parts := strings.Split(shard, "-")
if len(parts) != 2 {
return "", nil, fmt.Errorf("invalid shardId, can only contain one '-': %v", shard)
}
keyRange, err := key.ParseKeyRangeParts(parts[0], parts[1])
if err != nil {
return "", nil, err
}
if len(keyRange.End) > 0 && string(keyRange.Start) >= string(keyRange.End) {
return "", nil, fmt.Errorf("out of order keys: %v is not strictly smaller than %v", hex.EncodeToString(keyRange.Start), hex.EncodeToString(keyRange.End))
}
return strings.ToLower(shard), keyRange, nil
}
// ValidateShardName takes a shard name and sanitizes it, and also returns
// the KeyRange.
func ValidateShardName(shard string) (string, key.KeyRange, error) {
if !strings.Contains(shard, "-") {
return shard, key.KeyRange{}, nil
}
parts := strings.Split(shard, "-")
if len(parts) != 2 {
return "", key.KeyRange{}, fmt.Errorf("Invalid shardId, can only contain one '-': %v", shard)
}
keyRange, err := key.ParseKeyRangeParts(parts[0], parts[1])
if err != nil {
return "", key.KeyRange{}, err
}
if keyRange.End != key.MaxKey && keyRange.Start >= keyRange.End {
return "", key.KeyRange{}, fmt.Errorf("Out of order keys: %v is not strictly smaller than %v", keyRange.Start.Hex(), keyRange.End.Hex())
}
return strings.ToLower(shard), keyRange, nil
}
func TestVTGateStreamExecuteKeyRanges(t *testing.T) {
s := createSandbox("TestVTGateStreamExecuteKeyRanges")
sbc := &sandboxConn{}
s.MapTestConn("-20", sbc)
kr, err := key.ParseKeyRangeParts("", "20")
sq := proto.KeyRangeQuery{
Sql: "query",
Keyspace: "TestVTGateStreamExecuteKeyRanges",
KeyRanges: []key.KeyRange{kr},
TabletType: topo.TYPE_MASTER,
}
// Test for successful execution
var qrs []*proto.QueryResult
err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error {
qrs = append(qrs, r)
return nil
})
if err != nil {
t.Errorf("want nil, got %v", err)
}
row := new(proto.QueryResult)
proto.PopulateQueryResult(singleRowResult, row)
want := []*proto.QueryResult{row}
if !reflect.DeepEqual(want, qrs) {
t.Errorf("want \n%+v, got \n%+v", want, qrs)
}
sq.Session = new(proto.Session)
qrs = nil
RpcVTGate.Begin(nil, sq.Session)
err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error {
qrs = append(qrs, r)
return nil
})
want = []*proto.QueryResult{
row,
&proto.QueryResult{
Session: &proto.Session{
InTransaction: true,
ShardSessions: []*proto.ShardSession{{
Keyspace: "TestVTGateStreamExecuteKeyRanges",
Shard: "-20",
TransactionId: 1,
TabletType: topo.TYPE_MASTER,
}},
},
},
}
if !reflect.DeepEqual(want, qrs) {
t.Errorf("want \n%+v, got \n%+v", want, qrs)
}
// Test for error condition - multiple shards
kr, err = key.ParseKeyRangeParts("10", "40")
sq.KeyRanges = []key.KeyRange{kr}
err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error {
qrs = append(qrs, r)
return nil
})
if err == nil {
t.Errorf("want not nil, got %v", err)
}
// Test for error condition - multiple shards, non-partial keyspace
kr, err = key.ParseKeyRangeParts("", "")
sq.KeyRanges = []key.KeyRange{kr}
err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error {
qrs = append(qrs, r)
return nil
})
if err == nil {
t.Errorf("want not nil, got %v", err)
}
}
func TestVTGateExecuteKeyRanges(t *testing.T) {
s := createSandbox("TestVTGateExecuteKeyRanges")
sbc1 := &sandboxConn{}
sbc2 := &sandboxConn{}
s.MapTestConn("-20", sbc1)
s.MapTestConn("20-40", sbc2)
kr, err := key.ParseKeyRangeParts("", "20")
q := proto.KeyRangeQuery{
Sql: "query",
Keyspace: "TestVTGateExecuteKeyRanges",
KeyRanges: []key.KeyRange{kr},
TabletType: topo.TYPE_MASTER,
}
// Test for successful execution
qr := new(proto.QueryResult)
err = RpcVTGate.ExecuteKeyRanges(nil, &q, qr)
if err != nil {
t.Errorf("want nil, got %v", err)
}
wantqr := new(proto.QueryResult)
proto.PopulateQueryResult(singleRowResult, wantqr)
if !reflect.DeepEqual(wantqr, qr) {
t.Errorf("want \n%+v, got \n%+v", singleRowResult, qr)
}
if qr.Session != nil {
t.Errorf("want nil, got %+v\n", qr.Session)
}
if sbc1.ExecCount != 1 {
t.Errorf("want 1, got %v\n", sbc1.ExecCount)
}
// Test for successful execution in transaction
q.Session = new(proto.Session)
RpcVTGate.Begin(nil, q.Session)
if !q.Session.InTransaction {
t.Errorf("want true, got false")
}
err = RpcVTGate.ExecuteKeyRanges(nil, &q, qr)
if err != nil {
t.Errorf("want nil, got %v", err)
}
wantSession := &proto.Session{
InTransaction: true,
ShardSessions: []*proto.ShardSession{{
Keyspace: "TestVTGateExecuteKeyRanges",
Shard: "-20",
TransactionId: 1,
TabletType: topo.TYPE_MASTER,
}},
}
if !reflect.DeepEqual(wantSession, q.Session) {
t.Errorf("want \n%+v, got \n%+v", wantSession, q.Session)
}
RpcVTGate.Commit(nil, q.Session)
if sbc1.CommitCount.Get() != 1 {
t.Errorf("want 1, got %v", sbc1.CommitCount.Get())
}
// Test for multiple shards
kr, err = key.ParseKeyRangeParts("10", "30")
q.KeyRanges = []key.KeyRange{kr}
RpcVTGate.ExecuteKeyRanges(nil, &q, qr)
if qr.RowsAffected != 2 {
t.Errorf("want 2, got %v", qr.RowsAffected)
}
}
func TestVTGateExecuteKeyRanges(t *testing.T) {
s := createSandbox("TestVTGateExecuteKeyRanges")
sbc1 := &sandboxConn{}
sbc2 := &sandboxConn{}
s.MapTestConn("-20", sbc1)
s.MapTestConn("20-40", sbc2)
kr, err := key.ParseKeyRangeParts("", "20")
// Test for successful execution
qr := new(proto.QueryResult)
err = rpcVTGate.ExecuteKeyRanges(context.Background(),
"query",
nil,
"TestVTGateExecuteKeyRanges",
[]key.KeyRange{kr},
pb.TabletType_MASTER,
nil,
false,
qr)
if err != nil {
t.Errorf("want nil, got %v", err)
}
wantqr := new(proto.QueryResult)
wantqr.Result = singleRowResult
if !reflect.DeepEqual(wantqr, qr) {
t.Errorf("want \n%+v, got \n%+v", singleRowResult, qr)
}
if qr.Session != nil {
t.Errorf("want nil, got %+v\n", qr.Session)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v\n", execCount)
}
// Test for successful execution in transaction
session := new(proto.Session)
rpcVTGate.Begin(context.Background(), session)
if !session.InTransaction {
t.Errorf("want true, got false")
}
err = rpcVTGate.ExecuteKeyRanges(context.Background(),
"query",
nil,
"TestVTGateExecuteKeyRanges",
[]key.KeyRange{kr},
pb.TabletType_MASTER,
session,
false,
qr)
if err != nil {
t.Errorf("want nil, got %v", err)
}
wantSession := &proto.Session{
InTransaction: true,
ShardSessions: []*proto.ShardSession{{
Keyspace: "TestVTGateExecuteKeyRanges",
Shard: "-20",
TransactionId: 1,
TabletType: topo.TYPE_MASTER,
}},
}
if !reflect.DeepEqual(wantSession, session) {
t.Errorf("want \n%+v, got \n%+v", wantSession, session)
}
rpcVTGate.Commit(context.Background(), session)
if commitCount := sbc1.CommitCount.Get(); commitCount != 1 {
t.Errorf("want 1, got %v", commitCount)
}
// Test for multiple shards
kr, err = key.ParseKeyRangeParts("10", "30")
rpcVTGate.ExecuteKeyRanges(context.Background(), "query",
nil,
"TestVTGateExecuteKeyRanges",
[]key.KeyRange{kr},
pb.TabletType_MASTER,
nil,
false, qr)
if qr.Result.RowsAffected != 2 {
t.Errorf("want 2, got %v", qr.Result.RowsAffected)
}
}