func main() { flag.Parse() servenv.Init() defer servenv.Close() keyRange, err := key.ParseKeyRangeParts(*start, *end) if err != nil { log.Fatalf("Invalid key range: %v", err) } if *dbConfigFile == "" { log.Fatalf("Cannot start without db-config-file") } dbConfig, err := readDbConfig(*dbConfigFile) if err != nil { log.Fatalf("Cannot read db config file: %v", err) } var t []string if *tables != "" { t = strings.Split(*tables, ",") for i, table := range t { t[i] = strings.TrimSpace(table) } } interrupted := make(chan struct{}) c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGTERM) go func() { for _ = range c { close(interrupted) } }() var vtClient mysqlctl.VtClient vtClient = mysqlctl.NewDbClient(dbConfig) err = vtClient.Connect() if err != nil { log.Fatalf("error in initializing dbClient: %v", err) } brs, err := mysqlctl.ReadStartPosition(vtClient, uint32(*uid)) if err != nil { log.Fatalf("Cannot read start position from db: %v", err) } if *debug { vtClient = mysqlctl.NewDummyVtClient() } blp, err := mysqlctl.NewBinlogPlayer(vtClient, keyRange, uint32(*uid), brs, t, *txnBatch, time.Duration(*maxTxnInterval)*time.Second, *execDdl) if err != nil { log.Fatalf("error in initializing binlog player: %v", err) } err = blp.ApplyBinlogEvents(interrupted) if err != nil { log.Errorf("Error in applying binlog events, err %v", err) } log.Infof("vt_binlog_player done") }
func TestVTGateStreamExecuteKeyRanges(t *testing.T) { s := createSandbox("TestVTGateStreamExecuteKeyRanges") sbc := &sandboxConn{} s.MapTestConn("-20", sbc) sbc1 := &sandboxConn{} s.MapTestConn("20-40", sbc1) kr, err := key.ParseKeyRangeParts("", "20") // Test for successful execution var qrs []*proto.QueryResult err = rpcVTGate.StreamExecuteKeyRanges(context.Background(), "query", nil, "TestVTGateStreamExecuteKeyRanges", []key.KeyRange{kr}, pb.TabletType_MASTER, func(r *proto.QueryResult) error { qrs = append(qrs, r) return nil }) if err != nil { t.Errorf("want nil, got %v", err) } row := new(proto.QueryResult) row.Result = singleRowResult want := []*proto.QueryResult{row} if !reflect.DeepEqual(want, qrs) { t.Errorf("want \n%+v, got \n%+v", want, qrs) } // Test for successful execution - multiple shards kr, err = key.ParseKeyRangeParts("10", "40") err = rpcVTGate.StreamExecuteKeyRanges(context.Background(), "query", nil, "TestVTGateStreamExecuteKeyRanges", []key.KeyRange{kr}, pb.TabletType_MASTER, func(r *proto.QueryResult) error { qrs = append(qrs, r) return nil }) if err != nil { t.Errorf("want nil, got %v", err) } }
func multiRestoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { starts := subFlags.String("starts", "", "starts of the key range") ends := subFlags.String("ends", "", "ends of the key range") fetchRetryCount := subFlags.Int("fetch_retry_count", 3, "how many times to retry a failed transfer") concurrency := subFlags.Int("concurrency", 8, "how many concurrent db inserts to run simultaneously") fetchConcurrency := subFlags.Int("fetch_concurrency", 4, "how many files to fetch simultaneously") insertTableConcurrency := subFlags.Int("insert_table_concurrency", 4, "how many myisam tables to load into a single destination table simultaneously") strategy := subFlags.String("strategy", "", "which strategy to use for restore, can contain:\n"+ " skipAutoIncrement(TTT): we won't add the AUTO_INCREMENT back to that table\n"+ " delayPrimaryKey: we won't add the primary key until after the table is populated\n"+ " delaySecondaryIndexes: we won't add the secondary indexes until after the table is populated\n"+ " useMyIsam: create the table as MyISAM, then convert it to InnoDB after population\n"+ " writeBinLogs: write all operations to the binlogs") subFlags.Parse(args) if subFlags.NArg() < 2 { log.Fatalf("multirestore requires <destination_dbname> <source_host>[/<source_dbname>]... %v", args) } startArray := strings.Split(*starts, ",") endArray := strings.Split(*ends, ",") if len(startArray) != len(endArray) || len(startArray) != subFlags.NArg()-1 { log.Fatalf("Need as many starts and ends as source URLs") } keyRanges := make([]key.KeyRange, len(startArray)) for i, s := range startArray { var err error keyRanges[i], err = key.ParseKeyRangeParts(s, endArray[i]) if err != nil { log.Fatalf("Invalid start or end: %v", err) } } dbName, dbis := subFlags.Arg(0), subFlags.Args()[1:] sources := make([]*url.URL, len(dbis)) for i, dbi := range dbis { if !strings.HasPrefix(dbi, "vttp://") && !strings.HasPrefix(dbi, "http://") { dbi = "vttp://" + dbi } dbUrl, err := url.Parse(dbi) if err != nil { log.Fatalf("incorrect source url: %v", err) } sources[i] = dbUrl } if err := mysqld.MultiRestore(dbName, keyRanges, sources, nil, *concurrency, *fetchConcurrency, *insertTableConcurrency, *fetchRetryCount, *strategy); err != nil { log.Fatalf("multirestore failed: %v", err) } }
// ValidateShardName takes a shard name and sanitizes it, and also returns // the KeyRange. func ValidateShardName(shard string) (string, *topodatapb.KeyRange, error) { if !IsShardUsingRangeBasedSharding(shard) { return shard, nil, nil } parts := strings.Split(shard, "-") if len(parts) != 2 { return "", nil, fmt.Errorf("invalid shardId, can only contain one '-': %v", shard) } keyRange, err := key.ParseKeyRangeParts(parts[0], parts[1]) if err != nil { return "", nil, err } if len(keyRange.End) > 0 && string(keyRange.Start) >= string(keyRange.End) { return "", nil, fmt.Errorf("out of order keys: %v is not strictly smaller than %v", hex.EncodeToString(keyRange.Start), hex.EncodeToString(keyRange.End)) } return strings.ToLower(shard), keyRange, nil }
// ValidateShardName takes a shard name and sanitizes it, and also returns // the KeyRange. func ValidateShardName(shard string) (string, key.KeyRange, error) { if !strings.Contains(shard, "-") { return shard, key.KeyRange{}, nil } parts := strings.Split(shard, "-") if len(parts) != 2 { return "", key.KeyRange{}, fmt.Errorf("Invalid shardId, can only contain one '-': %v", shard) } keyRange, err := key.ParseKeyRangeParts(parts[0], parts[1]) if err != nil { return "", key.KeyRange{}, err } if keyRange.End != key.MaxKey && keyRange.Start >= keyRange.End { return "", key.KeyRange{}, fmt.Errorf("Out of order keys: %v is not strictly smaller than %v", keyRange.Start.Hex(), keyRange.End.Hex()) } return strings.ToLower(shard), keyRange, nil }
func TestVTGateStreamExecuteKeyRanges(t *testing.T) { s := createSandbox("TestVTGateStreamExecuteKeyRanges") sbc := &sandboxConn{} s.MapTestConn("-20", sbc) kr, err := key.ParseKeyRangeParts("", "20") sq := proto.KeyRangeQuery{ Sql: "query", Keyspace: "TestVTGateStreamExecuteKeyRanges", KeyRanges: []key.KeyRange{kr}, TabletType: topo.TYPE_MASTER, } // Test for successful execution var qrs []*proto.QueryResult err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error { qrs = append(qrs, r) return nil }) if err != nil { t.Errorf("want nil, got %v", err) } row := new(proto.QueryResult) proto.PopulateQueryResult(singleRowResult, row) want := []*proto.QueryResult{row} if !reflect.DeepEqual(want, qrs) { t.Errorf("want \n%+v, got \n%+v", want, qrs) } sq.Session = new(proto.Session) qrs = nil RpcVTGate.Begin(nil, sq.Session) err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error { qrs = append(qrs, r) return nil }) want = []*proto.QueryResult{ row, &proto.QueryResult{ Session: &proto.Session{ InTransaction: true, ShardSessions: []*proto.ShardSession{{ Keyspace: "TestVTGateStreamExecuteKeyRanges", Shard: "-20", TransactionId: 1, TabletType: topo.TYPE_MASTER, }}, }, }, } if !reflect.DeepEqual(want, qrs) { t.Errorf("want \n%+v, got \n%+v", want, qrs) } // Test for error condition - multiple shards kr, err = key.ParseKeyRangeParts("10", "40") sq.KeyRanges = []key.KeyRange{kr} err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error { qrs = append(qrs, r) return nil }) if err == nil { t.Errorf("want not nil, got %v", err) } // Test for error condition - multiple shards, non-partial keyspace kr, err = key.ParseKeyRangeParts("", "") sq.KeyRanges = []key.KeyRange{kr} err = RpcVTGate.StreamExecuteKeyRanges(nil, &sq, func(r *proto.QueryResult) error { qrs = append(qrs, r) return nil }) if err == nil { t.Errorf("want not nil, got %v", err) } }
func TestVTGateExecuteKeyRanges(t *testing.T) { s := createSandbox("TestVTGateExecuteKeyRanges") sbc1 := &sandboxConn{} sbc2 := &sandboxConn{} s.MapTestConn("-20", sbc1) s.MapTestConn("20-40", sbc2) kr, err := key.ParseKeyRangeParts("", "20") q := proto.KeyRangeQuery{ Sql: "query", Keyspace: "TestVTGateExecuteKeyRanges", KeyRanges: []key.KeyRange{kr}, TabletType: topo.TYPE_MASTER, } // Test for successful execution qr := new(proto.QueryResult) err = RpcVTGate.ExecuteKeyRanges(nil, &q, qr) if err != nil { t.Errorf("want nil, got %v", err) } wantqr := new(proto.QueryResult) proto.PopulateQueryResult(singleRowResult, wantqr) if !reflect.DeepEqual(wantqr, qr) { t.Errorf("want \n%+v, got \n%+v", singleRowResult, qr) } if qr.Session != nil { t.Errorf("want nil, got %+v\n", qr.Session) } if sbc1.ExecCount != 1 { t.Errorf("want 1, got %v\n", sbc1.ExecCount) } // Test for successful execution in transaction q.Session = new(proto.Session) RpcVTGate.Begin(nil, q.Session) if !q.Session.InTransaction { t.Errorf("want true, got false") } err = RpcVTGate.ExecuteKeyRanges(nil, &q, qr) if err != nil { t.Errorf("want nil, got %v", err) } wantSession := &proto.Session{ InTransaction: true, ShardSessions: []*proto.ShardSession{{ Keyspace: "TestVTGateExecuteKeyRanges", Shard: "-20", TransactionId: 1, TabletType: topo.TYPE_MASTER, }}, } if !reflect.DeepEqual(wantSession, q.Session) { t.Errorf("want \n%+v, got \n%+v", wantSession, q.Session) } RpcVTGate.Commit(nil, q.Session) if sbc1.CommitCount.Get() != 1 { t.Errorf("want 1, got %v", sbc1.CommitCount.Get()) } // Test for multiple shards kr, err = key.ParseKeyRangeParts("10", "30") q.KeyRanges = []key.KeyRange{kr} RpcVTGate.ExecuteKeyRanges(nil, &q, qr) if qr.RowsAffected != 2 { t.Errorf("want 2, got %v", qr.RowsAffected) } }
func TestVTGateExecuteKeyRanges(t *testing.T) { s := createSandbox("TestVTGateExecuteKeyRanges") sbc1 := &sandboxConn{} sbc2 := &sandboxConn{} s.MapTestConn("-20", sbc1) s.MapTestConn("20-40", sbc2) kr, err := key.ParseKeyRangeParts("", "20") // Test for successful execution qr := new(proto.QueryResult) err = rpcVTGate.ExecuteKeyRanges(context.Background(), "query", nil, "TestVTGateExecuteKeyRanges", []key.KeyRange{kr}, pb.TabletType_MASTER, nil, false, qr) if err != nil { t.Errorf("want nil, got %v", err) } wantqr := new(proto.QueryResult) wantqr.Result = singleRowResult if !reflect.DeepEqual(wantqr, qr) { t.Errorf("want \n%+v, got \n%+v", singleRowResult, qr) } if qr.Session != nil { t.Errorf("want nil, got %+v\n", qr.Session) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v\n", execCount) } // Test for successful execution in transaction session := new(proto.Session) rpcVTGate.Begin(context.Background(), session) if !session.InTransaction { t.Errorf("want true, got false") } err = rpcVTGate.ExecuteKeyRanges(context.Background(), "query", nil, "TestVTGateExecuteKeyRanges", []key.KeyRange{kr}, pb.TabletType_MASTER, session, false, qr) if err != nil { t.Errorf("want nil, got %v", err) } wantSession := &proto.Session{ InTransaction: true, ShardSessions: []*proto.ShardSession{{ Keyspace: "TestVTGateExecuteKeyRanges", Shard: "-20", TransactionId: 1, TabletType: topo.TYPE_MASTER, }}, } if !reflect.DeepEqual(wantSession, session) { t.Errorf("want \n%+v, got \n%+v", wantSession, session) } rpcVTGate.Commit(context.Background(), session) if commitCount := sbc1.CommitCount.Get(); commitCount != 1 { t.Errorf("want 1, got %v", commitCount) } // Test for multiple shards kr, err = key.ParseKeyRangeParts("10", "30") rpcVTGate.ExecuteKeyRanges(context.Background(), "query", nil, "TestVTGateExecuteKeyRanges", []key.KeyRange{kr}, pb.TabletType_MASTER, nil, false, qr) if qr.Result.RowsAffected != 2 { t.Errorf("want 2, got %v", qr.Result.RowsAffected) } }