func TestResolverDmlOnMultipleKeyspaceIds(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf("Error encoding keyspace id") } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf("Error encoding keyspace id") } res := NewResolver(new(sandboxTopo), "", "aa", retryDelay, 0, connTimeoutTotal, connTimeoutPerConn, connLife) s := createSandbox("TestResolverDmlOnMultipleKeyspaceIds") sbc0 := &sandboxConn{} s.MapTestConn("-20", sbc0) sbc1 := &sandboxConn{} s.MapTestConn("20-40", sbc1) errStr := "DML should not span multiple keyspace_ids" _, err = res.ExecuteKeyspaceIds(context.Background(), "update table set a = b", nil, "TestResolverExecuteKeyspaceIds", []key.KeyspaceId{kid10, kid25}, pb.TabletType_MASTER, nil, false) if err == nil { t.Errorf("want %v, got nil", errStr) } }
// ValidateShardName takes a shard name and sanitizes it, and also returns // the KeyRange. func ValidateShardName(shard string) (string, key.KeyRange, error) { if !strings.Contains(shard, "-") { return shard, key.KeyRange{}, nil } parts := strings.Split(shard, "-") if len(parts) != 2 { return "", key.KeyRange{}, fmt.Errorf("Invalid shardId, can only contain one '-': %v", shard) } start, err := key.HexKeyspaceId(parts[0]).Unhex() if err != nil { return "", key.KeyRange{}, err } end, err := key.HexKeyspaceId(parts[1]).Unhex() if err != nil { return "", key.KeyRange{}, err } if end != key.MaxKey && start >= end { return "", key.KeyRange{}, fmt.Errorf("Out of order keys: %v is not strictly smaller than %v", start, end) } return strings.ToUpper(shard), key.KeyRange{Start: start, End: end}, nil }
func TestResolverExecuteBatchKeyspaceIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteBatchKeyspaceIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } query := &proto.KeyspaceIdBatchQuery{ Queries: []proto.BoundKeyspaceIdQuery{{ Sql: "query", BindVariables: nil, Keyspace: "TestResolverExecuteBatchKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid25}, }}, TabletType: topo.TYPE_MASTER, AsTransaction: false, } res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) qrs, err := res.ExecuteBatchKeyspaceIds(context.Background(), query) if err != nil { return nil, err } return &qrs.List[0], err }) }
func TestResolverExecuteEntityIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteEntityIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) return res.ExecuteEntityIds(context.Background(), "query", nil, "TestResolverExecuteEntityIds", "col", []proto.EntityId{ proto.EntityId{ ExternalID: 0, KeyspaceID: kid10, }, proto.EntityId{ ExternalID: "1", KeyspaceID: kid25, }, }, pb.TabletType_MASTER, nil, false) }) }
func TestBoundKeyspaceIdQueriesToBoundShardQueries(t *testing.T) { ts := new(sandboxTopo) kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Error(err) } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Error(err) } var testCases = []struct { idQueries []proto.BoundKeyspaceIdQuery shardQueries []proto.BoundShardQuery }{ { idQueries: []proto.BoundKeyspaceIdQuery{ { Sql: "q1", BindVariables: map[string]interface{}{"q1var": 1}, Keyspace: KsTestSharded, KeyspaceIds: []key.KeyspaceId{kid10, kid25}, }, { Sql: "q2", BindVariables: map[string]interface{}{"q2var": 2}, Keyspace: KsTestSharded, KeyspaceIds: []key.KeyspaceId{kid25, kid25}, }, }, shardQueries: []proto.BoundShardQuery{ { Sql: "q1", BindVariables: map[string]interface{}{"q1var": 1}, Keyspace: KsTestSharded, Shards: []string{"-20", "20-40"}, }, { Sql: "q2", BindVariables: map[string]interface{}{"q2var": 2}, Keyspace: KsTestSharded, Shards: []string{"20-40"}, }, }, }, } for _, testCase := range testCases { shardQueries, err := boundKeyspaceIDQueriesToBoundShardQueries(context.Background(), ts, "", pb.TabletType_MASTER, testCase.idQueries) if err != nil { t.Error(err) } // Sort shards, because they're random otherwise. for _, shardQuery := range shardQueries { sort.Strings(shardQuery.Shards) } if !reflect.DeepEqual(testCase.shardQueries, shardQueries) { got, _ := json.Marshal(shardQueries) want, _ := json.Marshal(testCase.shardQueries) t.Errorf("idQueries: %#v\nResponse: %s\nExepecting: %s", testCase.idQueries, got, want) } } }
func TestResolverDmlOnMultipleKeyspaceIds(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf("Error encoding keyspace id") } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf("Error encoding keyspace id") } query := &proto.KeyspaceIdQuery{ Sql: "update table set a = b", Keyspace: "TestResolverExecuteKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid25}, TabletType: topo.TYPE_MASTER, } res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) s := createSandbox("TestResolverDmlOnMultipleKeyspaceIds") sbc0 := &sandboxConn{} s.MapTestConn("-20", sbc0) sbc1 := &sandboxConn{} s.MapTestConn("20-40", sbc1) errStr := "DML should not span multiple keyspace_ids" _, err = res.ExecuteKeyspaceIds(context.Background(), query) if err == nil { t.Errorf("want %v, got nil", errStr) } }
func TestResolverExecuteEntityIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteEntityIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } query := &proto.EntityIdsQuery{ Sql: "query", Keyspace: "TestResolverExecuteEntityIds", EntityColumnName: "col", EntityKeyspaceIDs: []proto.EntityId{ proto.EntityId{ ExternalID: 0, KeyspaceID: kid10, }, proto.EntityId{ ExternalID: "1", KeyspaceID: kid25, }, }, TabletType: topo.TYPE_MASTER, } res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) return res.ExecuteEntityIds(context.Background(), query) }) }
func TestResolverExecuteBatchKeyspaceIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteBatchKeyspaceIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } res := NewResolver(nil, topo.Server{}, new(sandboxTopo), "", "aa", retryDelay, 0, connTimeoutTotal, connTimeoutPerConn, connLife, "") qrs, err := res.ExecuteBatchKeyspaceIds(context.Background(), []proto.BoundKeyspaceIdQuery{{ Sql: "query", BindVariables: nil, Keyspace: "TestResolverExecuteBatchKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid25}, }}, pb.TabletType_MASTER, false, nil) if err != nil { return nil, err } return &qrs.List[0], err }) }
func NewBinlogPlayer(dbClient VtClient, startPosition *binlogRecoveryState, tables []string, txnBatch int, maxTxnInterval time.Duration, execDdl bool) (*BinlogPlayer, error) { if !startPositionValid(startPosition) { log.Fatalf("Invalid Start Position") } blp := new(BinlogPlayer) blp.recoveryState = *startPosition // convert start and end keyrange var err error blp.keyrange.Start, err = key.HexKeyspaceId(startPosition.KeyrangeStart).Unhex() if err != nil { return nil, fmt.Errorf("Error in Unhex for %v, '%v'", startPosition.KeyrangeStart, err) } blp.keyrange.End, err = key.HexKeyspaceId(startPosition.KeyrangeEnd).Unhex() if err != nil { return nil, fmt.Errorf("Error in Unhex for %v, '%v'", startPosition.KeyrangeEnd, err) } blp.txnIndex = 0 blp.inTxn = false blp.txnBuffer = make([]*cproto.BinlogResponse, 0, MAX_TXN_BATCH) blp.dbClient = dbClient blp.tables = tables blp.blplStats = NewBlplStats() blp.batchStart = time.Now() blp.txnBatch = txnBatch blp.maxTxnInterval = maxTxnInterval blp.execDdl = execDdl return blp, nil }
func (blServer *BinlogServer) ServeBinlog(req *mysqlctl.BinlogServerRequest, sendReply mysqlctl.SendUpdateStreamResponse) error { defer func() { if x := recover(); x != nil { //Send the error to the client. _, ok := x.(*BinlogParseError) if !ok { relog.Error("Uncaught panic at top-most level: '%v'", x) //panic(x) } sendError(sendReply, req.StartPosition.String(), x.(error), nil) } }() relog.Info("received req: %v kr start %v end %v", req.StartPosition.String(), req.KeyspaceStart, req.KeyspaceEnd) if !isRequestValid(req) { panic(NewBinlogParseError("Invalid request, cannot serve the stream")) } usingRelayLogs := false var binlogPrefix, logsDir string if req.StartPosition.RelayFilename != "" { usingRelayLogs = true binlogPrefix = blServer.mycnf.RelayLogPath logsDir = path.Dir(binlogPrefix) if !mysqlctl.IsRelayPositionValid(&req.StartPosition, logsDir) { panic(NewBinlogParseError(fmt.Sprintf("Invalid start position %v, cannot serve the stream, cannot locate start position", req.StartPosition))) } } else { binlogPrefix = blServer.mycnf.BinLogPath logsDir = path.Dir(binlogPrefix) if !mysqlctl.IsMasterPositionValid(&req.StartPosition) { panic(NewBinlogParseError(fmt.Sprintf("Invalid start position %v, cannot serve the stream, cannot locate start position", req.StartPosition))) } } startKey, err := key.HexKeyspaceId(req.KeyspaceStart).Unhex() if err != nil { panic(NewBinlogParseError(fmt.Sprintf("Unhex on key '%v' failed", req.KeyspaceStart))) } endKey, err := key.HexKeyspaceId(req.KeyspaceEnd).Unhex() if err != nil { panic(NewBinlogParseError(fmt.Sprintf("Unhex on key '%v' failed", req.KeyspaceEnd))) } keyRange := &key.KeyRange{Start: startKey, End: endKey} blp := NewBlp(&req.StartPosition, blServer, keyRange) blp.usingRelayLogs = usingRelayLogs blp.binlogPrefix = binlogPrefix blp.logMetadata = mysqlctl.NewSlaveMetadata(logsDir, blServer.mycnf.RelayLogInfoPath) relog.Info("usingRelayLogs %v blp.binlogPrefix %v logsDir %v", blp.usingRelayLogs, blp.binlogPrefix, logsDir) blp.streamBinlog(sendReply) return nil }
func TestVTGateExecuteBatchKeyspaceIds(t *testing.T) { //TODO(sougou): Fix test t.Skip() s := createSandbox("TestVTGateExecuteBatchKeyspaceIds") s.MapTestConn("-20", &sandboxConn{}) s.MapTestConn("20-40", &sandboxConn{}) kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf("want nil, got %v", err) } kid30, err := key.HexKeyspaceId("30").Unhex() if err != nil { t.Errorf("want nil, got %v", err) } q := proto.KeyspaceIdBatchQuery{ Queries: []proto.BoundKeyspaceIdQuery{{ Sql: "query", BindVariables: nil, Keyspace: "TestVTGateExecuteBatchKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid30}, }, { Sql: "query", BindVariables: nil, Keyspace: "TestVTGateExecuteBatchKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid30}, }}, TabletType: topo.TYPE_MASTER, } qrl := new(proto.QueryResultList) err = rpcVTGate.ExecuteBatchKeyspaceIds(context.Background(), &q, qrl) if err != nil { t.Errorf("want nil, got %v", err) } if len(qrl.List) != 2 { t.Errorf("want 2, got %v", len(qrl.List)) } if qrl.List[0].RowsAffected != 2 { t.Errorf("want 2, got %v", qrl.List[0].RowsAffected) } if qrl.Session != nil { t.Errorf("want nil, got %+v\n", qrl.Session) } q.Session = new(proto.Session) rpcVTGate.Begin(context.Background(), q.Session) rpcVTGate.ExecuteBatchKeyspaceIds(context.Background(), &q, qrl) if len(q.Session.ShardSessions) != 2 { t.Errorf("want 2, got %d", len(q.Session.ShardSessions)) } }
func hki(hexValue string) key.KeyspaceId { k, err := key.HexKeyspaceId(hexValue).Unhex() if err != nil { panic(err) } return k }
func hexOrDie(t *testing.T, hex string) key.KeyspaceId { kid, err := key.HexKeyspaceId(hex).Unhex() if err != nil { t.Fatalf("Unhex failed: %v", err) } return kid }
func TestResolverStreamExecuteKeyRanges(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf(err.Error()) } kid15, err := key.HexKeyspaceId("15").Unhex() if err != nil { t.Errorf(err.Error()) } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf(err.Error()) } createSandbox("TestResolverStreamExecuteKeyRanges") // streaming a single shard testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyRanges", func() (*mproto.QueryResult, error) { res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyRanges(context.Background(), "query", nil, "TestResolverStreamExecuteKeyRanges", []key.KeyRange{key.KeyRange{Start: kid10, End: kid15}}, pb.TabletType_MASTER, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) // streaming multiple shards testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyRanges", func() (*mproto.QueryResult, error) { res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyRanges(context.Background(), "query", nil, "TestResolverStreamExecuteKeyRanges", []key.KeyRange{key.KeyRange{Start: kid10, End: kid25}}, pb.TabletType_MASTER, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) }
func multiRestoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { start := subFlags.String("start", "", "start of the key range") end := subFlags.String("end", "", "end of the key range") fetchRetryCount := subFlags.Int("fetch-retry-count", 3, "how many times to retry a failed transfer") concurrency := subFlags.Int("concurrency", 8, "how many concurrent db inserts to run simultaneously") fetchConcurrency := subFlags.Int("fetch-concurrency", 4, "how many files to fetch simultaneously") insertTableConcurrency := subFlags.Int("insert-table-concurrency", 4, "how many myisam tables to load into a single destination table simultaneously") strategy := subFlags.String("strategy", "", "which strategy to use for restore, can contain:\n"+ " skipAutoIncrement(TTT): we won't add the AUTO_INCREMENT back to that table\n"+ " delayPrimaryKey: we won't add the primary key until after the table is populated\n"+ " delaySecondaryIndexes: we won't add the secondary indexes until after the table is populated\n"+ " useMyIsam: create the table as MyISAM, then convert it to InnoDB after population\n"+ " writeBinLogs: write all operations to the binlogs") subFlags.Parse(args) s, err := key.HexKeyspaceId(*start).Unhex() if err != nil { relog.Fatal("Invalid start key %v: %v", *start, err) } e, err := key.HexKeyspaceId(*end).Unhex() if err != nil { relog.Fatal("Invalid end key %v: %v", *end, err) } keyRange := key.KeyRange{Start: s, End: e} if subFlags.NArg() < 2 { relog.Fatal("multirestore requires <destination_dbname> <source_host>[/<source_dbname>]... %v", args) } dbName, dbis := subFlags.Arg(0), subFlags.Args()[1:] sources := make([]*url.URL, len(dbis)) uids := make([]uint32, len(dbis)) for i, dbi := range dbis { if !strings.HasPrefix(dbi, "vttp://") && !strings.HasPrefix(dbi, "http://") { dbi = "vttp://" + dbi } dbUrl, err := url.Parse(dbi) if err != nil { relog.Fatal("incorrect source url: %v", err) } sources[i] = dbUrl uids[i] = uint32(i) } if err := mysqld.RestoreFromMultiSnapshot(dbName, keyRange, sources, uids, *concurrency, *fetchConcurrency, *insertTableConcurrency, *fetchRetryCount, *strategy); err != nil { relog.Fatal("multirestore failed: %v", err) } }
func TestVTGateExecuteBatchKeyspaceIds(t *testing.T) { s := createSandbox("TestVTGateExecuteBatchKeyspaceIds") s.MapTestConn("-20", &sandboxConn{}) s.MapTestConn("20-40", &sandboxConn{}) kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf("want nil, got %v", err) } kid30, err := key.HexKeyspaceId("30").Unhex() if err != nil { t.Errorf("want nil, got %v", err) } q := proto.KeyspaceIdBatchQuery{ Queries: []tproto.BoundQuery{{ "query", nil, }, { "query", nil, }}, Keyspace: "TestVTGateExecuteBatchKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid30}, TabletType: topo.TYPE_MASTER, } qrl := new(proto.QueryResultList) err = RpcVTGate.ExecuteBatchKeyspaceIds(nil, &q, qrl) if err != nil { t.Errorf("want nil, got %v", err) } if len(qrl.List) != 2 { t.Errorf("want 2, got %v", len(qrl.List)) } if qrl.List[0].RowsAffected != 2 { t.Errorf("want 2, got %v", qrl.List[0].RowsAffected) } if qrl.Session != nil { t.Errorf("want nil, got %+v\n", qrl.Session) } q.Session = new(proto.Session) RpcVTGate.Begin(nil, q.Session) err = RpcVTGate.ExecuteBatchKeyspaceIds(nil, &q, qrl) if len(q.Session.ShardSessions) != 2 { t.Errorf("want 2, got %d", len(q.Session.ShardSessions)) } }
func TestResolverStreamExecuteKeyspaceIds(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf(err.Error()) } kid15, err := key.HexKeyspaceId("15").Unhex() if err != nil { t.Errorf(err.Error()) } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf(err.Error()) } createSandbox("TestResolverStreamExecuteKeyspaceIds") testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyspaceIds", func() (*mproto.QueryResult, error) { res := NewResolver(new(sandboxTopo), "", "aa", retryDelay, 0, connTimeoutTotal, connTimeoutPerConn, connLife) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyspaceIds(context.Background(), "query", nil, "TestResolverStreamExecuteKeyspaceIds", []key.KeyspaceId{kid10, kid15}, pb.TabletType_MASTER, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyspaceIds", func() (*mproto.QueryResult, error) { res := NewResolver(new(sandboxTopo), "", "aa", retryDelay, 0, connTimeoutTotal, connTimeoutPerConn, connLife) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyspaceIds(context.Background(), "query", nil, "TestResolverStreamExecuteKeyspaceIds", []key.KeyspaceId{kid10, kid15, kid25}, pb.TabletType_MASTER, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) }
func (blServer *BinlogServer) ServeBinlog(req *proto.BinlogServerRequest, sendReply proto.SendBinlogResponse) error { defer func() { if x := recover(); x != nil { //Send the error to the client. _, ok := x.(*BinlogServerError) if !ok { log.Errorf("Uncaught panic at top-most level: '%v'", x) //panic(x) } sendError(sendReply, req.StartPosition.String(), x.(error), nil) } }() log.Infof("received req: %v kr start %v end %v", req.StartPosition.String(), req.KeyspaceStart, req.KeyspaceEnd) if !blServer.isServiceEnabled() { panic(newBinlogServerError("Binlog Server is disabled")) } if !isRequestValid(req) { panic(newBinlogServerError("Invalid request, cannot serve the stream")) } binlogPrefix := blServer.mycnf.BinLogPath logsDir := path.Dir(binlogPrefix) if !IsMasterPositionValid(&req.StartPosition) { panic(newBinlogServerError(fmt.Sprintf("Invalid start position %v, cannot serve the stream, cannot locate start position", req.StartPosition))) } startKey, err := key.HexKeyspaceId(req.KeyspaceStart).Unhex() if err != nil { panic(newBinlogServerError(fmt.Sprintf("Unhex on key '%v' failed", req.KeyspaceStart))) } endKey, err := key.HexKeyspaceId(req.KeyspaceEnd).Unhex() if err != nil { panic(newBinlogServerError(fmt.Sprintf("Unhex on key '%v' failed", req.KeyspaceEnd))) } keyRange := &key.KeyRange{Start: startKey, End: endKey} blp := newBls(&req.StartPosition, blServer, keyRange) blp.binlogPrefix = binlogPrefix log.Infof("blp.binlogPrefix %v logsDir %v", blp.binlogPrefix, logsDir) blp.streamBinlog(sendReply, blServer.interrupted) return nil }
// ReadRecord returns a keyspaceId and a line from which it was // extracted, with the keyspaceId stripped. func (r KeyspaceCSVReader) ReadRecord() (keyspaceId key.KeyspaceId, line []byte, err error) { k, err := r.reader.ReadString(r.delim) if err != nil { return key.MinKey, nil, err } if r.numberColumn { // the line starts with: // NNNN, // so remove the comma kid, err := strconv.ParseUint(k[:len(k)-1], 10, 64) if err != nil { return key.MinKey, nil, err } keyspaceId = key.Uint64Key(kid).KeyspaceId() } else { // the line starts with: // "HHHH", // so remove the quotes and comma keyspaceId, err = key.HexKeyspaceId(k[1 : len(k)-2]).Unhex() if err != nil { return key.MinKey, nil, err } } defer r.buf.Reset() escaped := false inQuote := false for { b, err := r.reader.ReadByte() if err != nil { // Assumption: the csv file ends with a // newline. Otherwise io.EOF should be treated // separately. return key.MinKey, nil, err } r.buf.WriteByte(b) if escaped { escaped = false continue } switch b { case '\\': escaped = true case '"': inQuote = !inQuote case '\n': if !inQuote { return keyspaceId, r.buf.Bytes(), nil } } } }
func TestResolverStreamExecuteKeyRanges(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf(err.Error()) } kid15, err := key.HexKeyspaceId("15").Unhex() if err != nil { t.Errorf(err.Error()) } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf(err.Error()) } query := &proto.KeyRangeQuery{ Sql: "query", Keyspace: "TestResolverStreamExecuteKeyRanges", KeyRanges: []key.KeyRange{key.KeyRange{Start: kid10, End: kid15}}, TabletType: topo.TYPE_MASTER, } createSandbox("TestResolverStreamExecuteKeyRanges") // streaming a single shard testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyRanges", func() (*mproto.QueryResult, error) { res := NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyRanges(nil, query, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) // streaming multiple shards testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyRanges", func() (*mproto.QueryResult, error) { query.KeyRanges = []key.KeyRange{key.KeyRange{Start: kid10, End: kid25}} res := NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyRanges(nil, query, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) }
func TestResolverExecuteKeyRanges(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteKeyRanges", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } query := &proto.KeyRangeQuery{ Sql: "query", Keyspace: "TestResolverExecuteKeyRanges", KeyRanges: []key.KeyRange{key.KeyRange{Start: kid10, End: kid25}}, TabletType: topo.TYPE_MASTER, } res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) return res.ExecuteKeyRanges(context.Background(), query) }) }
func TestResolverExecuteKeyspaceIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteKeyspaceIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } query := &proto.KeyspaceIdQuery{ Sql: "query", Keyspace: "TestResolverExecuteKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid25}, TabletType: topo.TYPE_MASTER, } res := NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) return res.ExecuteKeyspaceIds(nil, query) }) }
func TestResolverExecuteKeyRanges(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteKeyRanges", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } res := NewResolver(new(sandboxTopo), "", "aa", retryDelay, 0, connTimeoutTotal, connTimeoutPerConn, connLife) return res.ExecuteKeyRanges(context.Background(), "query", nil, "TestResolverExecuteKeyRanges", []key.KeyRange{key.KeyRange{Start: kid10, End: kid25}}, pb.TabletType_MASTER, nil, false) }) }
func TestResolverStreamExecuteKeyspaceIds(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf(err.Error()) } kid15, err := key.HexKeyspaceId("15").Unhex() if err != nil { t.Errorf(err.Error()) } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf(err.Error()) } query := &proto.KeyspaceIdQuery{ Sql: "query", Keyspace: "TestResolverStreamExecuteKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid15}, TabletType: topo.TYPE_MASTER, } createSandbox("TestResolverStreamExecuteKeyspaceIds") testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyspaceIds", func() (*mproto.QueryResult, error) { res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyspaceIds(context.Background(), query, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyspaceIds", func() (*mproto.QueryResult, error) { query.KeyspaceIds = []key.KeyspaceId{kid10, kid15, kid25} res := NewResolver(new(sandboxTopo), "", "aa", 1*time.Millisecond, 0, 2*time.Millisecond, 1*time.Millisecond, 24*time.Hour) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyspaceIds(context.Background(), query, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) }
func TestResolverStreamExecuteKeyspaceIds(t *testing.T) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { t.Errorf(err.Error()) } kid15, err := key.HexKeyspaceId("15").Unhex() if err != nil { t.Errorf(err.Error()) } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { t.Errorf(err.Error()) } query := &proto.KeyspaceIdQuery{ Sql: "query", Keyspace: "TestResolverStreamExecuteKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid15, kid25}, TabletType: topo.TYPE_MASTER, } createSandbox("TestResolverStreamExecuteKeyspaceIds") res := NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) err = res.StreamExecuteKeyspaceIds(nil, query, func(r *mproto.QueryResult) error { return nil }) want := "Resolved to more than one shard" if err == nil || err.Error() != want { t.Errorf("want %s, got %v", want, err) } testResolverStreamGeneric(t, "TestResolverStreamExecuteKeyspaceIds", func() (*mproto.QueryResult, error) { query.KeyspaceIds = []key.KeyspaceId{kid10, kid15} res = NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) qr := new(mproto.QueryResult) err = res.StreamExecuteKeyspaceIds(nil, query, func(r *mproto.QueryResult) error { appendResult(qr, r) return nil }) return qr, err }) }
func TestResolverExecuteEntityIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteEntityIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } query := &proto.EntityIdsQuery{ Sql: "query", Keyspace: "TestResolverExecuteEntityIds", EntityColumnName: "col", EntityKeyspaceIdMap: map[string]key.KeyspaceId{ "0": kid10, "1": kid25, }, TabletType: topo.TYPE_MASTER, } res := NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) return res.ExecuteEntityIds(nil, query) }) }
func TestResolverExecuteBatchKeyspaceIds(t *testing.T) { testResolverGeneric(t, "TestResolverExecuteBatchKeyspaceIds", func() (*mproto.QueryResult, error) { kid10, err := key.HexKeyspaceId("10").Unhex() if err != nil { return nil, err } kid25, err := key.HexKeyspaceId("25").Unhex() if err != nil { return nil, err } query := &proto.KeyspaceIdBatchQuery{ Queries: []tproto.BoundQuery{{"query", nil}}, Keyspace: "TestResolverExecuteBatchKeyspaceIds", KeyspaceIds: []key.KeyspaceId{kid10, kid25}, TabletType: topo.TYPE_MASTER, } res := NewResolver(new(sandboxTopo), "aa", 1*time.Millisecond, 0, 1*time.Millisecond) qrs, err := res.ExecuteBatchKeyspaceIds(nil, query) if err != nil { return nil, err } return &qrs.List[0], err }) }
func partialSnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { start := subFlags.String("start", "", "start of the key range") end := subFlags.String("end", "", "end of the key range") concurrency := subFlags.Int("concurrency", 4, "how many compression jobs to run simultaneously") subFlags.Parse(args) if subFlags.NArg() != 2 { relog.Fatal("action partialsnapshot requires <db name> <key name>") } filename, err := mysqld.CreateSplitSnapshot(subFlags.Arg(0), subFlags.Arg(1), key.HexKeyspaceId(*start), key.HexKeyspaceId(*end), tabletAddr, false, *concurrency, nil) if err != nil { relog.Fatal("partialsnapshot failed: %v", err) } else { relog.Info("manifest location: %v", filename) } }
func TestUint64FromKeyspaceId(t *testing.T) { table := map[string]string{ "10": "0x1000000000000000", "fe": "0xfe00000000000000", "1234cafe": "0x1234cafe00000000", } for input, want := range table { keyspaceID, err := key.HexKeyspaceId(input).Unhex() if err != nil { t.Errorf("Unhex error: %v", err) continue } if got := uint64FromKeyspaceId(keyspaceID); got != want { t.Errorf("uint64FromKeyspaceId(%v) = %q, want %q", input, got, want) } } }
func initBinlogPlayer(startPosFile, dbConfigFile, lookupConfigFile, dbCredFile string, useCheckpoint, debug bool, port int) (*BinlogPlayer, error) { startData, err := ioutil.ReadFile(startPosFile) if err != nil { return nil, fmt.Errorf("Error %s in reading start position file %s", err, startPosFile) } startPosition := new(binlogRecoveryState) err = json.Unmarshal(startData, startPosition) if err != nil { return nil, fmt.Errorf("Error in unmarshaling recovery data: %s, startData %v", err, string(startData)) } dbClient, err := createDbClient(dbConfigFile) if err != nil { return nil, err } if useCheckpoint { selectRecovery := fmt.Sprintf(SELECT_FROM_RECOVERY, startPosition.Uid) qr, err := dbClient.ExecuteFetch(selectRecovery, 1, true) if err != nil { panic(fmt.Errorf("Error %v in selecting from recovery table %v", err, selectRecovery)) } if qr.RowsAffected != 1 { relog.Fatal("Checkpoint information not available in db") } startCoord, err := getStartPosition(qr) if err != nil { relog.Fatal("Error in obtaining checkpoint information") } startPosition.Position = *startCoord } if !startPositionValid(startPosition) { return nil, fmt.Errorf("Invalid Start Position") } krStart, err := key.HexKeyspaceId(startPosition.KeyrangeStart).Unhex() if err != nil { return nil, fmt.Errorf("Error in Unhex for %v, '%v'", startPosition.KeyrangeStart, err) } krEnd, err := key.HexKeyspaceId(startPosition.KeyrangeEnd).Unhex() if err != nil { return nil, fmt.Errorf("Error in Unhex for %v, '%v'", startPosition.KeyrangeEnd, err) } binlogPlayer := NewBinlogPlayer(startPosition, port, krStart, krEnd) if debug { binlogPlayer.debug = true binlogPlayer.dbClient = dummyVtClient{} binlogPlayer.lookupClient = dummyVtClient{} } else { binlogPlayer.dbClient = *dbClient lookupClient, err := createLookupClient(lookupConfigFile, dbCredFile) if err != nil { return nil, err } binlogPlayer.lookupClient = *lookupClient if !useCheckpoint { initialize_recovery_table(dbClient, startPosition, port) } useDb := fmt.Sprintf(USE_DB, dbClient.dbConfig.Dbname) binlogPlayer.updatePort(port, startPosition.Uid, useDb) } return binlogPlayer, nil }