// DialTablet creates and initializes TabletBson.
func DialTablet(ctx context.Context, endPoint topo.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) {
var addr string
var config *tls.Config
if *tabletBsonEncrypted {
addr = netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["vts"])
config = &tls.Config{}
config.InsecureSkipVerify = true
} else {
addr = netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["vt"])
}
conn := &TabletBson{endPoint: endPoint}
var err error
if *tabletBsonUsername != "" {
conn.rpcClient, err = bsonrpc.DialAuthHTTP("tcp", addr, *tabletBsonUsername, *tabletBsonPassword, timeout, config)
} else {
conn.rpcClient, err = bsonrpc.DialHTTP("tcp", addr, timeout, config)
}
if err != nil {
return nil, tabletError(err)
}
var sessionInfo tproto.SessionInfo
if err = conn.rpcClient.Call(ctx, "SqlQuery.GetSessionId", tproto.SessionParams{Keyspace: keyspace, Shard: shard}, &sessionInfo); err != nil {
conn.rpcClient.Close()
return nil, tabletError(err)
}
// SqlQuery.GetSessionId might return an application error inside the SessionInfo
if err = vterrors.FromRPCError(sessionInfo.Err); err != nil {
conn.rpcClient.Close()
return nil, tabletError(err)
}
conn.sessionID = sessionInfo.SessionId
return conn, nil
}
// ShortName returns a displayable representation of the host name.
// If the host is an IP address instead of a name, it is not shortened.
func (tn *TabletNode) ShortName() string {
if net.ParseIP(tn.Host) != nil {
return netutil.JoinHostPort(tn.Host, tn.Port)
}
hostPart := strings.SplitN(tn.Host, ".", 2)[0]
if tn.Port == 0 {
return hostPart
}
return netutil.JoinHostPort(hostPart, tn.Port)
}
// StatusAsHTML returns an HTML version of the status.
func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML {
tLinks := make([]string, 0, 1)
if tcs.TabletsStats != nil {
sort.Sort(tcs.TabletsStats)
}
for _, ts := range tcs.TabletsStats {
vtPort := ts.Tablet.PortMap["vt"]
color := "green"
extra := ""
if ts.LastError != nil {
color = "red"
extra = fmt.Sprintf(" (%v)", ts.LastError)
} else if !ts.Serving {
color = "red"
extra = " (Not Serving)"
} else if !ts.Up {
color = "red"
extra = " (Down)"
} else if ts.Target.TabletType == topodatapb.TabletType_MASTER {
extra = fmt.Sprintf(" (MasterTS: %v)", ts.TabletExternallyReparentedTimestamp)
} else {
extra = fmt.Sprintf(" (RepLag: %v)", ts.Stats.SecondsBehindMaster)
}
name := ts.Name
addr := netutil.JoinHostPort(ts.Tablet.Hostname, vtPort)
if name == "" {
name = addr
}
tLinks = append(tLinks, fmt.Sprintf(`<a href="http://%s" style="color:%v">%v</a>%v`, addr, color, name, extra))
}
return template.HTML(strings.Join(tLinks, "<br>"))
}
// DialTablet creates and initializes gRPCQueryClient.
func DialTablet(ctx context.Context, endPoint topo.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) {
// create the RPC client
addr := netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["grpc"])
cc, err := grpc.Dial(addr)
if err != nil {
return nil, err
}
c := pbs.NewQueryClient(cc)
gsir, err := c.GetSessionId(ctx, &pb.GetSessionIdRequest{
Keyspace: keyspace,
Shard: shard,
})
if err != nil {
cc.Close()
return nil, err
}
if gsir.Error != nil {
cc.Close()
return nil, tabletErrorFromRPCError(gsir.Error)
}
return &gRPCQueryClient{
endPoint: endPoint,
cc: cc,
c: c,
sessionID: gsir.SessionId,
}, nil
}
// DialTablet creates and initializes TabletBson.
func DialTablet(ctx context.Context, endPoint *pbt.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) {
addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["vt"])
conn := &TabletBson{endPoint: endPoint}
var err error
if *tabletBsonUsername != "" {
conn.rpcClient, err = bsonrpc.DialAuthHTTP("tcp", addr, *tabletBsonUsername, *tabletBsonPassword, timeout)
} else {
conn.rpcClient, err = bsonrpc.DialHTTP("tcp", addr, timeout)
}
if err != nil {
return nil, tabletError(err)
}
if keyspace != "" || shard != "" {
var sessionInfo tproto.SessionInfo
if err = conn.rpcClient.Call(ctx, "SqlQuery.GetSessionId", tproto.SessionParams{Keyspace: keyspace, Shard: shard}, &sessionInfo); err != nil {
conn.rpcClient.Close()
return nil, tabletError(err)
}
// SqlQuery.GetSessionId might return an application error inside the SessionInfo
if err = vterrors.FromRPCError(sessionInfo.Err); err != nil {
conn.rpcClient.Close()
return nil, tabletError(err)
}
conn.sessionID = sessionInfo.SessionId
}
return conn, nil
}
// DialTablet creates and initializes gRPCQueryClient.
func DialTablet(tablet *topodatapb.Tablet, timeout time.Duration) (tabletconn.TabletConn, error) {
// create the RPC client
addr := ""
if grpcPort, ok := tablet.PortMap["grpc"]; ok {
addr = netutil.JoinHostPort(tablet.Hostname, grpcPort)
} else {
addr = tablet.Hostname
}
opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name)
if err != nil {
return nil, err
}
opts := []grpc.DialOption{opt}
if timeout > 0 {
opts = append(opts, grpc.WithBlock(), grpc.WithTimeout(timeout))
}
cc, err := grpc.Dial(addr, opts...)
if err != nil {
return nil, err
}
c := queryservicepb.NewQueryClient(cc)
result := &gRPCQueryClient{
tablet: tablet,
cc: cc,
c: c,
}
return result, nil
}
// DialTablet creates and initializes gRPCQueryClient.
func DialTablet(ctx context.Context, tablet *topodatapb.Tablet, timeout time.Duration) (tabletconn.TabletConn, error) {
// create the RPC client
addr := netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["grpc"])
opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name)
if err != nil {
return nil, err
}
cc, err := grpc.Dial(addr, opt, grpc.WithBlock(), grpc.WithTimeout(timeout))
if err != nil {
return nil, err
}
c := queryservicepb.NewQueryClient(cc)
result := &gRPCQueryClient{
tablet: tablet,
cc: cc,
c: c,
target: &querypb.Target{
Keyspace: tablet.Keyspace,
Shard: tablet.Shard,
TabletType: tablet.Type,
},
}
return result, nil
}
func (client *Client) dialPool(tablet *topodatapb.Tablet) (tabletmanagerservicepb.TabletManagerClient, error) {
addr := netutil.JoinHostPort(tablet.Hostname, int32(tablet.PortMap["grpc"]))
opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name)
if err != nil {
return nil, err
}
client.mu.Lock()
if client.rpcClientMap == nil {
client.rpcClientMap = make(map[string]chan *tmc)
}
c, ok := client.rpcClientMap[addr]
if !ok {
c = make(chan *tmc, *concurrency)
client.rpcClientMap[addr] = c
client.mu.Unlock()
for i := 0; i < cap(c); i++ {
cc, err := grpc.Dial(addr, opt)
if err != nil {
return nil, err
}
c <- &tmc{
cc: cc,
client: tabletmanagerservicepb.NewTabletManagerClient(cc),
}
}
} else {
client.mu.Unlock()
}
result := <-c
c <- result
return result.client, nil
}
// DialTablet creates and initializes gRPCQueryClient.
func DialTablet(ctx context.Context, endPoint *pbt.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) {
// create the RPC client
addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["grpc"])
cc, err := grpc.Dial(addr, grpc.WithBlock(), grpc.WithTimeout(timeout))
if err != nil {
return nil, err
}
c := pbs.NewQueryClient(cc)
result := &gRPCQueryClient{
endPoint: endPoint,
cc: cc,
c: c,
}
if keyspace != "" || shard != "" {
gsir, err := c.GetSessionId(ctx, &pb.GetSessionIdRequest{
Keyspace: keyspace,
Shard: shard,
})
if err != nil {
cc.Close()
return nil, err
}
if gsir.Error != nil {
cc.Close()
return nil, tabletErrorFromRPCError(gsir.Error)
}
result.sessionID = gsir.SessionId
}
return result, nil
}
// TabletToMapKey creates a key to the map from tablet's host and ports.
// It should only be used in discovery and related module.
func TabletToMapKey(tablet *topodatapb.Tablet) string {
parts := make([]string, 0, 1)
for name, port := range tablet.PortMap {
parts = append(parts, netutil.JoinHostPort(name, port))
}
sort.Strings(parts)
parts = append([]string{tablet.Hostname}, parts...)
return strings.Join(parts, ",")
}
func (client *client) Dial(endPoint *topodatapb.EndPoint, connTimeout time.Duration) error {
addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["grpc"])
var err error
client.cc, err = grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(connTimeout))
if err != nil {
return err
}
client.c = binlogservicepb.NewUpdateStreamClient(client.cc)
return nil
}
func addTabletLinks(result *explorerResult, data string) {
t := &topo.Tablet{}
err := json.Unmarshal([]byte(data), t)
if err != nil {
return
}
if port, ok := t.Portmap["vt"]; ok {
result.Links["status"] = template.URL(fmt.Sprintf("http://%v/debug/status", netutil.JoinHostPort(t.Hostname, int32(port))))
}
}
func (client *client) Dial(endPoint topo.EndPoint, connTimeout time.Duration) error {
addr := netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["grpc"])
var err error
client.cc, err = grpc.Dial(addr, grpc.WithBlock(), grpc.WithTimeout(connTimeout))
if err != nil {
return err
}
client.c = pbs.NewUpdateStreamClient(client.cc)
client.ctx = context.Background()
return nil
}
func (ex ZkExplorer) addTabletLinks(data string, result *ZkResult) {
t := &pb.Tablet{}
err := json.Unmarshal([]byte(data), t)
if err != nil {
return
}
if port, ok := t.PortMap["vt"]; ok {
result.Links["status"] = template.URL(fmt.Sprintf("http://%v/debug/status", netutil.JoinHostPort(t.Hostname, port)))
}
}
// dial returns a client to use
func (client *Client) dial(tablet *topodatapb.Tablet) (*grpc.ClientConn, tabletmanagerservicepb.TabletManagerClient, error) {
addr := netutil.JoinHostPort(tablet.Hostname, int32(tablet.PortMap["grpc"]))
opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name)
if err != nil {
return nil, nil, err
}
cc, err := grpc.Dial(addr, opt)
if err != nil {
return nil, nil, err
}
return cc, tabletmanagerservicepb.NewTabletManagerClient(cc), nil
}
func populateListeningURL() {
host, err := netutil.FullyQualifiedHostname()
if err != nil {
host, err = os.Hostname()
if err != nil {
log.Fatalf("os.Hostname() failed: %v", err)
}
}
ListeningURL = url.URL{
Scheme: "http",
Host: netutil.JoinHostPort(host, int32(*Port)),
Path: "/",
}
}
func main() {
defer exit.Recover()
defer logutil.Flush()
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [global parameters] command [command parameters]\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\nThe global optional parameters are:\n")
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, "\nThe commands are listed below. Use '%s <command> -h' for more help.\n\n", os.Args[0])
for _, cmd := range commands {
fmt.Fprintf(os.Stderr, " %s", cmd.name)
if cmd.params != "" {
fmt.Fprintf(os.Stderr, " %s", cmd.params)
}
fmt.Fprintf(os.Stderr, "\n")
}
fmt.Fprintf(os.Stderr, "\n")
}
dbconfigs.RegisterFlags(dbconfigFlags)
flag.Parse()
tabletAddr = netutil.JoinHostPort("localhost", int32(*port))
action := flag.Arg(0)
for _, cmd := range commands {
if cmd.name == action {
subFlags := flag.NewFlagSet(action, flag.ExitOnError)
subFlags.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s %s %s\n\n", os.Args[0], cmd.name, cmd.params)
fmt.Fprintf(os.Stderr, "%s\n\n", cmd.help)
subFlags.PrintDefaults()
}
if err := cmd.method(subFlags, flag.Args()[1:]); err != nil {
log.Error(err)
exit.Return(1)
}
return
}
}
log.Errorf("invalid action: %v", action)
exit.Return(1)
}
// dial returns a client to use
func (client *Client) dial(ctx context.Context, tablet *topo.TabletInfo) (*grpc.ClientConn, pbs.TabletManagerClient, error) {
// create the RPC client, using ctx.Deadline if set, or no timeout.
var connectTimeout time.Duration
deadline, ok := ctx.Deadline()
if ok {
connectTimeout = deadline.Sub(time.Now())
if connectTimeout < 0 {
return nil, nil, timeoutError{fmt.Errorf("timeout connecting to TabletManager on %v", tablet.Alias)}
}
}
var cc *grpc.ClientConn
var err error
addr := netutil.JoinHostPort(tablet.Hostname, tablet.Portmap["grpc"])
if connectTimeout == 0 {
cc, err = grpc.Dial(addr, grpc.WithBlock())
} else {
cc, err = grpc.Dial(addr, grpc.WithBlock(), grpc.WithTimeout(connectTimeout))
}
if err != nil {
return nil, nil, err
}
return cc, pbs.NewTabletManagerClient(cc), nil
}
// DialTablet creates and initializes gRPCQueryClient.
func DialTablet(ctx context.Context, endPoint *pbt.EndPoint, keyspace, shard string, tabletType pbt.TabletType, timeout time.Duration) (tabletconn.TabletConn, error) {
// create the RPC client
addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["grpc"])
cc, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(timeout))
if err != nil {
return nil, err
}
c := pbs.NewQueryClient(cc)
result := &gRPCQueryClient{
endPoint: endPoint,
cc: cc,
c: c,
}
if tabletType == pbt.TabletType_UNKNOWN {
// we use session
gsir, err := c.GetSessionId(ctx, &pb.GetSessionIdRequest{
Keyspace: keyspace,
Shard: shard,
})
if err != nil {
cc.Close()
return nil, err
}
result.sessionID = gsir.SessionId
} else {
// we use target
result.target = &pb.Target{
Keyspace: keyspace,
Shard: shard,
TabletType: tabletType,
}
}
return result, nil
}
func (client *client) Dial(endPoint *pb.EndPoint, connTimeout time.Duration) error {
addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["vt"])
var err error
client.Client, err = bsonrpc.DialHTTP("tcp", addr, connTimeout)
return err
}
// MasterAddr returns the host:port address of the master.
func (rs *ReplicationStatus) MasterAddr() string {
return netutil.JoinHostPort(rs.MasterHost, int32(rs.MasterPort))
}
// Addr returns the fully qualified host name + port for this instance.
func (mysqld *Mysqld) Addr() string {
hostname := netutil.FullyQualifiedHostnameOrPanic()
return netutil.JoinHostPort(hostname, int32(mysqld.config.MysqlPort))
}
func main() {
defer exit.Recover()
defer logutil.Flush()
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [global parameters] command [command parameters]\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\nThe global optional parameters are:\n")
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, "\nThe commands are listed below. Use '%s <command> -h' for more help.\n\n", os.Args[0])
for _, cmd := range commands {
fmt.Fprintf(os.Stderr, " %s", cmd.name)
if cmd.params != "" {
fmt.Fprintf(os.Stderr, " %s", cmd.params)
}
fmt.Fprintf(os.Stderr, "\n")
}
fmt.Fprintf(os.Stderr, "\n")
}
flags := dbconfigs.AppConfig | dbconfigs.DbaConfig |
dbconfigs.FilteredConfig | dbconfigs.ReplConfig
dbconfigs.RegisterFlags(flags)
flag.Parse()
tabletAddr = netutil.JoinHostPort("localhost", int32(*port))
mycnf := mysqlctl.NewMycnf(uint32(*tabletUID), *mysqlPort)
if *mysqlSocket != "" {
mycnf.SocketFile = *mysqlSocket
}
dbcfgs, err := dbconfigs.Init(mycnf.SocketFile, flags)
if err != nil {
log.Errorf("%v", err)
exit.Return(1)
}
mysqld := mysqlctl.NewMysqld("Dba", "App", mycnf, &dbcfgs.Dba, &dbcfgs.App.ConnParams, &dbcfgs.Repl)
defer mysqld.Close()
action := flag.Arg(0)
for _, cmd := range commands {
if cmd.name == action {
subFlags := flag.NewFlagSet(action, flag.ExitOnError)
subFlags.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s %s %s\n\n", os.Args[0], cmd.name, cmd.params)
fmt.Fprintf(os.Stderr, "%s\n\n", cmd.help)
subFlags.PrintDefaults()
}
if err := cmd.method(mysqld, subFlags, flag.Args()[1:]); err != nil {
log.Error(err)
exit.Return(1)
}
return
}
}
log.Errorf("invalid action: %v", action)
exit.Return(1)
}
// TabletAddr returns hostname:vt port associated with a tablet
func TabletAddr(tablet *topodatapb.Tablet) string {
return netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["vt"])
}
// MysqlAddr returns hostname:mysql port.
func (ti *TabletInfo) MysqlAddr() string {
return netutil.JoinHostPort(ti.Hostname, int32(ti.PortMap["mysql"]))
}
// MysqlIPAddr returns ip:mysql port.
func (tablet *Tablet) MysqlIPAddr() string {
return netutil.JoinHostPort(tablet.IPAddr, int32(tablet.Portmap["mysql"]))
}
// Addr returns hostname:vt port.
func (tablet *Tablet) Addr() string {
return netutil.JoinHostPort(tablet.Hostname, int32(tablet.Portmap["vt"]))
}
// Iteration is a single iteration for the player: get the current status,
// try to play, and plays until interrupted, or until an error occurs.
func (bpc *BinlogPlayerController) Iteration(ctx context.Context) (err error) {
defer func() {
if x := recover(); x != nil {
log.Errorf("%v: caught panic: %v", bpc, x)
err = fmt.Errorf("panic: %v", x)
}
}()
// Apply any special settings necessary for playback of binlogs.
// We do it on every iteration to be sure, in case MySQL was restarted.
if err := bpc.mysqld.EnableBinlogPlayback(); err != nil {
// We failed to apply the required settings, so we shouldn't keep going.
return err
}
// create the db connection, connect it
vtClient := binlogplayer.NewDbClient(bpc.dbConfig)
if err := vtClient.Connect(); err != nil {
return fmt.Errorf("can't connect to database: %v", err)
}
defer vtClient.Close()
// Read the start position
startPosition, flags, err := binlogplayer.ReadStartPosition(vtClient, bpc.sourceShard.Uid)
if err != nil {
return fmt.Errorf("can't read startPosition: %v", err)
}
// if we shouldn't start, we just error out and try again later
if strings.Index(flags, binlogplayer.BlpFlagDontStart) != -1 {
return fmt.Errorf("not starting because flag '%v' is set", binlogplayer.BlpFlagDontStart)
}
// Find the server list for the source shard in our cell
addrs, _, err := bpc.ts.GetEndPoints(ctx, bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topo.TYPE_REPLICA)
if err != nil {
return fmt.Errorf("can't find any source tablet for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA, err)
}
if len(addrs.Entries) == 0 {
return fmt.Errorf("empty source tablet list for %v %v %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA)
}
newServerIndex := rand.Intn(len(addrs.Entries))
port, _ := addrs.Entries[newServerIndex].NamedPortMap["vt"]
addr := netutil.JoinHostPort(addrs.Entries[newServerIndex].Host, port)
// save our current server
bpc.playerMutex.Lock()
bpc.sourceTablet = topo.TabletAlias{
Cell: bpc.cell,
Uid: addrs.Entries[newServerIndex].Uid,
}
bpc.lastError = nil
bpc.playerMutex.Unlock()
// check which kind of replication we're doing, tables or keyrange
if len(bpc.sourceShard.Tables) > 0 {
// tables, first resolve wildcards
tables, err := mysqlctl.ResolveTables(bpc.mysqld, bpc.dbName, bpc.sourceShard.Tables)
if err != nil {
return fmt.Errorf("failed to resolve table names: %v", err)
}
// tables, just get them
player := binlogplayer.NewBinlogPlayerTables(vtClient, addr, tables, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
return player.ApplyBinlogEvents(bpc.interrupted)
}
// the data we have to replicate is the intersection of the
// source keyrange and our keyrange
overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange)
if err != nil {
return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange)
}
player := binlogplayer.NewBinlogPlayerKeyRange(vtClient, addr, bpc.keyspaceIDType, overlap, startPosition, bpc.stopPosition, bpc.binlogPlayerStats)
return player.ApplyBinlogEvents(bpc.interrupted)
}