// This function implements the restriction of handling one keyrange // and one shard since streaming doesn't support merge sorting the results. // The input/output api is generic though. func (vtg *VTGate) mapKrToShardsForStreaming(streamQuery *proto.StreamQueryKeyRange) ([]string, error) { var keyRange key.KeyRange var err error if streamQuery.KeyRange == "" { keyRange = key.KeyRange{Start: "", End: ""} } else { krArray, err := key.ParseShardingSpec(streamQuery.KeyRange) if err != nil { return nil, err } keyRange = krArray[0] } shards, err := resolveKeyRangeToShards(vtg.scatterConn.toposerv, vtg.scatterConn.cell, streamQuery.Keyspace, streamQuery.TabletType, keyRange) if err != nil { return nil, err } if len(shards) != 1 { return nil, fmt.Errorf("KeyRange cannot map to more than one shard") } return shards, nil }
func getAllShards(shardSpec string) (key.KeyRangeArray, error) { shardedKrArray, err := key.ParseShardingSpec(shardSpec) if err != nil { return nil, err } return shardedKrArray, nil }
func multisnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { concurrency := subFlags.Int("concurrency", 8, "how many compression jobs to run simultaneously") spec := subFlags.String("spec", "-", "shard specification") tablesString := subFlags.String("tables", "", "dump only this comma separated list of tables") skipSlaveRestart := subFlags.Bool("skip-slave-restart", false, "after the snapshot is done, do not restart slave replication") maximumFilesize := subFlags.Uint64("maximum-file-size", 128*1024*1024, "the maximum size for an uncompressed data file") subFlags.Parse(args) if subFlags.NArg() != 2 { relog.Fatal("action multisnapshot requires <db name> <key name>") } shards, err := key.ParseShardingSpec(*spec) if err != nil { relog.Fatal("multisnapshot failed: %v", err) } var tables []string if *tablesString != "" { tables = strings.Split(*tablesString, ",") } filenames, err := mysqld.CreateMultiSnapshot(shards, subFlags.Arg(0), subFlags.Arg(1), tabletAddr, false, *concurrency, tables, *skipSlaveRestart, *maximumFilesize, nil) if err != nil { relog.Fatal("multisnapshot failed: %v", err) } else { relog.Info("manifest locations: %v", filenames) } }
func TestVTGateSplitQuery(t *testing.T) { keyspace := "TestVTGateSplitQuery" keyranges, _ := key.ParseShardingSpec(DefaultShardSpec) createSandbox(keyspace) hcVTGateTest.Reset() port := int32(1001) for _, kr := range keyranges { hcVTGateTest.AddTestTablet("aa", "1.1.1.1", port, keyspace, key.KeyRangeString(kr), topodatapb.TabletType_RDONLY, true, 1, nil) port++ } sql := "select col1, col2 from table" splitCount := 24 splits, err := rpcVTGate.SplitQuery(context.Background(), keyspace, sql, nil, "", int64(splitCount)) if err != nil { t.Errorf("want nil, got %v", err) } _, err = getAllShards(DefaultShardSpec) // Total number of splits should be number of shards * splitsPerShard if splitCount != len(splits) { t.Errorf("wrong number of splits, want \n%+v, got \n%+v", splitCount, len(splits)) } actualSqlsByKeyRange := map[string][]string{} for _, split := range splits { if split.Size != sandboxconn.SandboxSQRowCount { t.Errorf("wrong split size, want \n%+v, got \n%+v", sandboxconn.SandboxSQRowCount, split.Size) } if split.KeyRangePart.Keyspace != keyspace { t.Errorf("wrong keyspace, want \n%+v, got \n%+v", keyspace, split.KeyRangePart.Keyspace) } if len(split.KeyRangePart.KeyRanges) != 1 { t.Errorf("wrong number of keyranges, want \n%+v, got \n%+v", 1, len(split.KeyRangePart.KeyRanges)) } kr := key.KeyRangeString(split.KeyRangePart.KeyRanges[0]) actualSqlsByKeyRange[kr] = append(actualSqlsByKeyRange[kr], split.Query.Sql) } // Sort the sqls for each KeyRange so that we can compare them without // regard to the order in which they were returned by the vtgate. for _, sqlsForKeyRange := range actualSqlsByKeyRange { sort.Strings(sqlsForKeyRange) } expectedSqlsByKeyRange := map[string][]string{} for _, kr := range keyranges { expectedSqlsByKeyRange[key.KeyRangeString(kr)] = []string{ "select col1, col2 from table /*split 0 */", "select col1, col2 from table /*split 1 */", "select col1, col2 from table /*split 2 */", } } if !reflect.DeepEqual(actualSqlsByKeyRange, expectedSqlsByKeyRange) { t.Errorf("splits contain the wrong sqls and/or keyranges, got: %v, want: %v", actualSqlsByKeyRange, expectedSqlsByKeyRange) } }
func getAllShards() (key.KeyRangeArray, error) { if ShardedKrArray != nil { return ShardedKrArray, nil } shardedKrArray, err := key.ParseShardingSpec(ShardSpec) if err != nil { return nil, err } return shardedKrArray, nil }
func TestVTGateSplitQuery(t *testing.T) { keyspace := "TestVTGateSplitQuery" keyranges, _ := key.ParseShardingSpec(DefaultShardSpec) s := createSandbox(keyspace) for _, kr := range keyranges { s.MapTestConn(fmt.Sprintf("%s-%s", kr.Start, kr.End), &sandboxConn{}) } sql := "select col1, col2 from table" splitCount := 24 req := proto.SplitQueryRequest{ Keyspace: keyspace, Query: tproto.BoundQuery{ Sql: sql, }, SplitCount: splitCount, } result := new(proto.SplitQueryResult) err := rpcVTGate.SplitQuery(context.Background(), &req, result) if err != nil { t.Errorf("want nil, got %v", err) } _, err = getAllShards(DefaultShardSpec) // Total number of splits should be number of shards * splitsPerShard if splitCount != len(result.Splits) { t.Errorf("wrong number of splits, want \n%+v, got \n%+v", splitCount, len(result.Splits)) } actualSqlsByKeyRange := map[kproto.KeyRange][]string{} for _, split := range result.Splits { if split.Size != sandboxSQRowCount { t.Errorf("wrong split size, want \n%+v, got \n%+v", sandboxSQRowCount, split.Size) } if split.Query.Keyspace != keyspace { t.Errorf("wrong split size, want \n%+v, got \n%+v", keyspace, split.Query.Keyspace) } if len(split.Query.KeyRanges) != 1 { t.Errorf("wrong number of keyranges, want \n%+v, got \n%+v", 1, len(split.Query.KeyRanges)) } if split.Query.TabletType != topo.TYPE_RDONLY { t.Errorf("wrong tablet type, want \n%+v, got \n%+v", topo.TYPE_RDONLY, split.Query.TabletType) } kr := split.Query.KeyRanges[0] actualSqlsByKeyRange[kr] = append(actualSqlsByKeyRange[kr], split.Query.Sql) } expectedSqlsByKeyRange := map[kproto.KeyRange][]string{} for _, kr := range keyranges { expectedSqlsByKeyRange[kr] = []string{ "select col1, col2 from table /*split 0 */", "select col1, col2 from table /*split 1 */", "select col1, col2 from table /*split 2 */", } } if !reflect.DeepEqual(actualSqlsByKeyRange, expectedSqlsByKeyRange) { t.Errorf("splits contain the wrong sqls and/or keyranges, got: %v, want: %v", actualSqlsByKeyRange, expectedSqlsByKeyRange) } }
func TestVTGateSplitQuery(t *testing.T) { keyspace := "TestVTGateSplitQuery" keyranges, _ := key.ParseShardingSpec(DefaultShardSpec) s := createSandbox(keyspace) for _, kr := range keyranges { s.MapTestConn(key.KeyRangeString(kr), &sandboxConn{}) } sql := "select col1, col2 from table" splitCount := 24 splits, err := rpcVTGate.SplitQuery(context.Background(), keyspace, sql, nil, "", splitCount) if err != nil { t.Errorf("want nil, got %v", err) } _, err = getAllShards(DefaultShardSpec) // Total number of splits should be number of shards * splitsPerShard if splitCount != len(splits) { t.Errorf("wrong number of splits, want \n%+v, got \n%+v", splitCount, len(splits)) } actualSqlsByKeyRange := map[string][]string{} for _, split := range splits { if split.Size != sandboxSQRowCount { t.Errorf("wrong split size, want \n%+v, got \n%+v", sandboxSQRowCount, split.Size) } if split.KeyRangePart.Keyspace != keyspace { t.Errorf("wrong split size, want \n%+v, got \n%+v", keyspace, split.KeyRangePart.Keyspace) } if len(split.KeyRangePart.KeyRanges) != 1 { t.Errorf("wrong number of keyranges, want \n%+v, got \n%+v", 1, len(split.KeyRangePart.KeyRanges)) } kr := key.KeyRangeString(split.KeyRangePart.KeyRanges[0]) actualSqlsByKeyRange[kr] = append(actualSqlsByKeyRange[kr], split.Query.Sql) } expectedSqlsByKeyRange := map[string][]string{} for _, kr := range keyranges { expectedSqlsByKeyRange[key.KeyRangeString(kr)] = []string{ "select col1, col2 from table /*split 0 */", "select col1, col2 from table /*split 1 */", "select col1, col2 from table /*split 2 */", } } if !reflect.DeepEqual(actualSqlsByKeyRange, expectedSqlsByKeyRange) { t.Errorf("splits contain the wrong sqls and/or keyranges, got: %v, want: %v", actualSqlsByKeyRange, expectedSqlsByKeyRange) } }
func TestMapKeyRangesToShards(t *testing.T) { ts := new(sandboxTopo) var testCases = []struct { keyspace string keyRange string shards []string }{ {keyspace: KsTestSharded, keyRange: "20-40", shards: []string{"20-40"}}, // check for partial keyrange, spanning one shard {keyspace: KsTestSharded, keyRange: "10-18", shards: []string{"-20"}}, // check for keyrange intersecting with multiple shards {keyspace: KsTestSharded, keyRange: "10-40", shards: []string{"-20", "20-40"}}, // check for keyrange intersecting with multiple shards {keyspace: KsTestSharded, keyRange: "1c-2a", shards: []string{"-20", "20-40"}}, // check for keyrange where kr.End is Max Key "" {keyspace: KsTestSharded, keyRange: "80-", shards: []string{"80-a0", "a0-c0", "c0-e0", "e0-"}}, // test for sharded, non-partial keyrange spanning the entire space. {keyspace: KsTestSharded, keyRange: "", shards: []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}}, // test for unsharded, non-partial keyrange spanning the entire space. {keyspace: KsTestUnsharded, keyRange: "", shards: []string{"0"}}, } for _, testCase := range testCases { var keyRange *topodatapb.KeyRange var err error if testCase.keyRange == "" { keyRange = &topodatapb.KeyRange{} } else { krArray, err := key.ParseShardingSpec(testCase.keyRange) if err != nil { t.Errorf("Got error while parsing sharding spec %v", err) } keyRange = krArray[0] } krs := []*topodatapb.KeyRange{keyRange} _, gotShards, err := mapKeyRangesToShards(context.Background(), ts, "", testCase.keyspace, topodatapb.TabletType_MASTER, krs) if err != nil { t.Errorf("want nil, got %v", err) } sort.Strings(gotShards) if !reflect.DeepEqual(testCase.shards, gotShards) { t.Errorf("want \n%#v, got \n%#v", testCase.shards, gotShards) } } }
func TestKeyRangeToShardMap(t *testing.T) { ts := new(sandboxTopo) var testCases = []struct { keyspace string keyRange string shards []string }{ {keyspace: TEST_SHARDED, keyRange: "20-40", shards: []string{"20-40"}}, // check for partial keyrange, spanning one shard {keyspace: TEST_SHARDED, keyRange: "10-18", shards: []string{"-20"}}, // check for keyrange intersecting with multiple shards {keyspace: TEST_SHARDED, keyRange: "10-40", shards: []string{"-20", "20-40"}}, // check for keyrange intersecting with multiple shards {keyspace: TEST_SHARDED, keyRange: "1C-2A", shards: []string{"-20", "20-40"}}, // check for keyrange where kr.End is Max Key "" {keyspace: TEST_SHARDED, keyRange: "80-", shards: []string{"80-A0", "A0-C0", "C0-E0", "E0-"}}, // test for sharded, non-partial keyrange spanning the entire space. {keyspace: TEST_SHARDED, keyRange: "", shards: []string{"-20", "20-40", "40-60", "60-80", "80-A0", "A0-C0", "C0-E0", "E0-"}}, // test for unsharded, non-partial keyrange spanning the entire space. {keyspace: TEST_UNSHARDED, keyRange: "", shards: []string{"0"}}, } for _, testCase := range testCases { var keyRange key.KeyRange var err error if testCase.keyRange == "" { keyRange = key.KeyRange{Start: "", End: ""} } else { krArray, err := key.ParseShardingSpec(testCase.keyRange) if err != nil { t.Errorf("Got error while parsing sharding spec %v", err) } keyRange = krArray[0] } allShards, err := getKeyspaceShards(ts, "", testCase.keyspace, topo.TYPE_MASTER) gotShards, err := resolveKeyRangeToShards(allShards, keyRange) if err != nil { t.Errorf("want nil, got %v", err) } if !reflect.DeepEqual(testCase.shards, gotShards) { t.Errorf("want \n%#v, got \n%#v", testCase.shards, gotShards) } } }
func TestMapExactShards(t *testing.T) { ts := new(sandboxTopo) var testCases = []struct { keyspace string keyRange string shards []string err string }{ {keyspace: KsTestSharded, keyRange: "20-40", shards: []string{"20-40"}}, // check for partial keyrange, spanning one shard {keyspace: KsTestSharded, keyRange: "10-18", shards: nil, err: "keyrange {Start: 10, End: 18} does not exactly match shards"}, // check for keyrange intersecting with multiple shards {keyspace: KsTestSharded, keyRange: "10-40", shards: nil, err: "keyrange {Start: 10, End: 40} does not exactly match shards"}, // check for keyrange intersecting with multiple shards {keyspace: KsTestSharded, keyRange: "1c-2a", shards: nil, err: "keyrange {Start: 1c, End: 2a} does not exactly match shards"}, // check for keyrange where kr.End is Max Key "" {keyspace: KsTestSharded, keyRange: "80-", shards: []string{"80-a0", "a0-c0", "c0-e0", "e0-"}}, // test for sharded, non-partial keyrange spanning the entire space. {keyspace: KsTestSharded, keyRange: "", shards: []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}}, } for _, testCase := range testCases { var keyRange key.KeyRange var err error if testCase.keyRange == "" { keyRange = key.KeyRange{Start: "", End: ""} } else { krArray, err := key.ParseShardingSpec(testCase.keyRange) if err != nil { t.Errorf("Got error while parsing sharding spec %v", err) } keyRange = krArray[0] } _, gotShards, err := mapExactShards(context.Background(), ts, "", testCase.keyspace, pb.TabletType_MASTER, keyRange) if err != nil && err.Error() != testCase.err { t.Errorf("gotShards: %v, want %s", err, testCase.err) } if !reflect.DeepEqual(testCase.shards, gotShards) { t.Errorf("want \n%#v, got \n%#v", testCase.shards, gotShards) } } }
func multisnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { concurrency := subFlags.Int("concurrency", 8, "how many compression jobs to run simultaneously") spec := subFlags.String("spec", "-", "shard specification") tablesString := subFlags.String("tables", "", "dump only this comma separated list of regexp for tables") excludeTablesString := subFlags.String("exclude_tables", "", "do not dump this comma separated list of regexp for tables") skipSlaveRestart := subFlags.Bool("skip_slave_restart", false, "after the snapshot is done, do not restart slave replication") maximumFilesize := subFlags.Uint64("maximum_file_size", 128*1024*1024, "the maximum size for an uncompressed data file") keyType := subFlags.String("key_type", "uint64", "type of the key column") subFlags.Parse(args) if subFlags.NArg() != 2 { log.Fatalf("action multisnapshot requires <db name> <key name>") } shards, err := key.ParseShardingSpec(*spec) if err != nil { log.Fatalf("multisnapshot failed: %v", err) } var tables []string if *tablesString != "" { tables = strings.Split(*tablesString, ",") } var excludedTables []string if *excludeTablesString != "" { excludedTables = strings.Split(*excludeTablesString, ",") } kit := key.KeyspaceIdType(*keyType) if !key.IsKeyspaceIdTypeInList(kit, key.AllKeyspaceIdTypes) { log.Fatalf("invalid key_type") } filenames, err := mysqld.CreateMultiSnapshot(shards, subFlags.Arg(0), subFlags.Arg(1), kit, tabletAddr, false, *concurrency, tables, excludedTables, *skipSlaveRestart, *maximumFilesize, nil) if err != nil { log.Fatalf("multisnapshot failed: %v", err) } else { log.Infof("manifest locations: %v", filenames) } }
// TODO(erez): Rename after migration to SplitQuery V2 is done. func TestVTGateSplitQueryV2Sharded(t *testing.T) { keyspace := "TestVTGateSplitQuery" keyranges, err := key.ParseShardingSpec(DefaultShardSpec) if err != nil { t.Fatalf("got: %v, want: nil", err) } createSandbox(keyspace) hcVTGateTest.Reset() port := int32(1001) for _, kr := range keyranges { hcVTGateTest.AddTestTablet("aa", "1.1.1.1", port, keyspace, key.KeyRangeString(kr), topodatapb.TabletType_RDONLY, true, 1, nil) port++ } sql := "select col1, col2 from table" bindVars := map[string]interface{}{"bv1": nil} splitColumns := []string{"sc1", "sc2"} algorithm := querypb.SplitQueryRequest_FULL_SCAN type testCaseType struct { splitCount int64 numRowsPerQueryPart int64 } testCases := []testCaseType{ {splitCount: 100, numRowsPerQueryPart: 0}, {splitCount: 0, numRowsPerQueryPart: 123}, } for _, testCase := range testCases { splits, err := rpcVTGate.SplitQueryV2( context.Background(), keyspace, sql, bindVars, splitColumns, testCase.splitCount, testCase.numRowsPerQueryPart, algorithm) if err != nil { t.Errorf("got %v, want: nil. testCase: %+v", err, testCase) } // Total number of splits should be number of shards as our sandbox returns a single split // for its fake implementation of SplitQuery. if len(keyranges) != len(splits) { t.Errorf("wrong number of splits, got %+v, want %+v. testCase:\n%+v", len(splits), len(keyranges), testCase) } actualSqlsByKeyRange := map[string][]string{} for _, split := range splits { if split.KeyRangePart.Keyspace != keyspace { t.Errorf("wrong keyspace, got \n%+v, want \n%+v. testCase:\n%+v", keyspace, split.KeyRangePart.Keyspace, testCase) } if len(split.KeyRangePart.KeyRanges) != 1 { t.Errorf("wrong number of keyranges, got \n%+v, want \n%+v. testCase:\n%+v", 1, len(split.KeyRangePart.KeyRanges), testCase) } kr := key.KeyRangeString(split.KeyRangePart.KeyRanges[0]) actualSqlsByKeyRange[kr] = append(actualSqlsByKeyRange[kr], split.Query.Sql) } expectedSqlsByKeyRange := map[string][]string{} for _, kr := range keyranges { perShardSplitCount := int64(math.Ceil(float64(testCase.splitCount) / float64(len(keyranges)))) shard := key.KeyRangeString(kr) expectedSqlsByKeyRange[shard] = []string{ fmt.Sprintf( "query:%v, splitColumns:%v, splitCount:%v,"+ " numRowsPerQueryPart:%v, algorithm:%v, shard:%v", querytypes.BoundQuery{Sql: sql, BindVariables: bindVars}, splitColumns, perShardSplitCount, testCase.numRowsPerQueryPart, algorithm, shard, ), } } if !reflect.DeepEqual(actualSqlsByKeyRange, expectedSqlsByKeyRange) { t.Errorf( "splits contain the wrong sqls and/or keyranges, "+ "got:\n%+v\n, want:\n%+v\n. testCase:\n%+v", actualSqlsByKeyRange, expectedSqlsByKeyRange, testCase) } } }