// DialTablet creates and initializes TabletBson. func DialTablet(ctx context.Context, endPoint topo.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) { var addr string var config *tls.Config if *tabletBsonEncrypted { addr = netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["vts"]) config = &tls.Config{} config.InsecureSkipVerify = true } else { addr = netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["vt"]) } conn := &TabletBson{endPoint: endPoint} var err error if *tabletBsonUsername != "" { conn.rpcClient, err = bsonrpc.DialAuthHTTP("tcp", addr, *tabletBsonUsername, *tabletBsonPassword, timeout, config) } else { conn.rpcClient, err = bsonrpc.DialHTTP("tcp", addr, timeout, config) } if err != nil { return nil, tabletError(err) } var sessionInfo tproto.SessionInfo if err = conn.rpcClient.Call(ctx, "SqlQuery.GetSessionId", tproto.SessionParams{Keyspace: keyspace, Shard: shard}, &sessionInfo); err != nil { conn.rpcClient.Close() return nil, tabletError(err) } // SqlQuery.GetSessionId might return an application error inside the SessionInfo if err = vterrors.FromRPCError(sessionInfo.Err); err != nil { conn.rpcClient.Close() return nil, tabletError(err) } conn.sessionID = sessionInfo.SessionId return conn, nil }
// ShortName returns a displayable representation of the host name. // If the host is an IP address instead of a name, it is not shortened. func (tn *TabletNode) ShortName() string { if net.ParseIP(tn.Host) != nil { return netutil.JoinHostPort(tn.Host, tn.Port) } hostPart := strings.SplitN(tn.Host, ".", 2)[0] if tn.Port == 0 { return hostPart } return netutil.JoinHostPort(hostPart, tn.Port) }
// StatusAsHTML returns an HTML version of the status. func (tcs *TabletsCacheStatus) StatusAsHTML() template.HTML { tLinks := make([]string, 0, 1) if tcs.TabletsStats != nil { sort.Sort(tcs.TabletsStats) } for _, ts := range tcs.TabletsStats { vtPort := ts.Tablet.PortMap["vt"] color := "green" extra := "" if ts.LastError != nil { color = "red" extra = fmt.Sprintf(" (%v)", ts.LastError) } else if !ts.Serving { color = "red" extra = " (Not Serving)" } else if !ts.Up { color = "red" extra = " (Down)" } else if ts.Target.TabletType == topodatapb.TabletType_MASTER { extra = fmt.Sprintf(" (MasterTS: %v)", ts.TabletExternallyReparentedTimestamp) } else { extra = fmt.Sprintf(" (RepLag: %v)", ts.Stats.SecondsBehindMaster) } name := ts.Name addr := netutil.JoinHostPort(ts.Tablet.Hostname, vtPort) if name == "" { name = addr } tLinks = append(tLinks, fmt.Sprintf(`<a href="http://%s" style="color:%v">%v</a>%v`, addr, color, name, extra)) } return template.HTML(strings.Join(tLinks, "<br>")) }
// DialTablet creates and initializes gRPCQueryClient. func DialTablet(ctx context.Context, endPoint topo.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) { // create the RPC client addr := netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["grpc"]) cc, err := grpc.Dial(addr) if err != nil { return nil, err } c := pbs.NewQueryClient(cc) gsir, err := c.GetSessionId(ctx, &pb.GetSessionIdRequest{ Keyspace: keyspace, Shard: shard, }) if err != nil { cc.Close() return nil, err } if gsir.Error != nil { cc.Close() return nil, tabletErrorFromRPCError(gsir.Error) } return &gRPCQueryClient{ endPoint: endPoint, cc: cc, c: c, sessionID: gsir.SessionId, }, nil }
// DialTablet creates and initializes TabletBson. func DialTablet(ctx context.Context, endPoint *pbt.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) { addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["vt"]) conn := &TabletBson{endPoint: endPoint} var err error if *tabletBsonUsername != "" { conn.rpcClient, err = bsonrpc.DialAuthHTTP("tcp", addr, *tabletBsonUsername, *tabletBsonPassword, timeout) } else { conn.rpcClient, err = bsonrpc.DialHTTP("tcp", addr, timeout) } if err != nil { return nil, tabletError(err) } if keyspace != "" || shard != "" { var sessionInfo tproto.SessionInfo if err = conn.rpcClient.Call(ctx, "SqlQuery.GetSessionId", tproto.SessionParams{Keyspace: keyspace, Shard: shard}, &sessionInfo); err != nil { conn.rpcClient.Close() return nil, tabletError(err) } // SqlQuery.GetSessionId might return an application error inside the SessionInfo if err = vterrors.FromRPCError(sessionInfo.Err); err != nil { conn.rpcClient.Close() return nil, tabletError(err) } conn.sessionID = sessionInfo.SessionId } return conn, nil }
// DialTablet creates and initializes gRPCQueryClient. func DialTablet(tablet *topodatapb.Tablet, timeout time.Duration) (tabletconn.TabletConn, error) { // create the RPC client addr := "" if grpcPort, ok := tablet.PortMap["grpc"]; ok { addr = netutil.JoinHostPort(tablet.Hostname, grpcPort) } else { addr = tablet.Hostname } opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name) if err != nil { return nil, err } opts := []grpc.DialOption{opt} if timeout > 0 { opts = append(opts, grpc.WithBlock(), grpc.WithTimeout(timeout)) } cc, err := grpc.Dial(addr, opts...) if err != nil { return nil, err } c := queryservicepb.NewQueryClient(cc) result := &gRPCQueryClient{ tablet: tablet, cc: cc, c: c, } return result, nil }
// DialTablet creates and initializes gRPCQueryClient. func DialTablet(ctx context.Context, tablet *topodatapb.Tablet, timeout time.Duration) (tabletconn.TabletConn, error) { // create the RPC client addr := netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["grpc"]) opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name) if err != nil { return nil, err } cc, err := grpc.Dial(addr, opt, grpc.WithBlock(), grpc.WithTimeout(timeout)) if err != nil { return nil, err } c := queryservicepb.NewQueryClient(cc) result := &gRPCQueryClient{ tablet: tablet, cc: cc, c: c, target: &querypb.Target{ Keyspace: tablet.Keyspace, Shard: tablet.Shard, TabletType: tablet.Type, }, } return result, nil }
func (client *Client) dialPool(tablet *topodatapb.Tablet) (tabletmanagerservicepb.TabletManagerClient, error) { addr := netutil.JoinHostPort(tablet.Hostname, int32(tablet.PortMap["grpc"])) opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name) if err != nil { return nil, err } client.mu.Lock() if client.rpcClientMap == nil { client.rpcClientMap = make(map[string]chan *tmc) } c, ok := client.rpcClientMap[addr] if !ok { c = make(chan *tmc, *concurrency) client.rpcClientMap[addr] = c client.mu.Unlock() for i := 0; i < cap(c); i++ { cc, err := grpc.Dial(addr, opt) if err != nil { return nil, err } c <- &tmc{ cc: cc, client: tabletmanagerservicepb.NewTabletManagerClient(cc), } } } else { client.mu.Unlock() } result := <-c c <- result return result.client, nil }
// DialTablet creates and initializes gRPCQueryClient. func DialTablet(ctx context.Context, endPoint *pbt.EndPoint, keyspace, shard string, timeout time.Duration) (tabletconn.TabletConn, error) { // create the RPC client addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["grpc"]) cc, err := grpc.Dial(addr, grpc.WithBlock(), grpc.WithTimeout(timeout)) if err != nil { return nil, err } c := pbs.NewQueryClient(cc) result := &gRPCQueryClient{ endPoint: endPoint, cc: cc, c: c, } if keyspace != "" || shard != "" { gsir, err := c.GetSessionId(ctx, &pb.GetSessionIdRequest{ Keyspace: keyspace, Shard: shard, }) if err != nil { cc.Close() return nil, err } if gsir.Error != nil { cc.Close() return nil, tabletErrorFromRPCError(gsir.Error) } result.sessionID = gsir.SessionId } return result, nil }
// TabletToMapKey creates a key to the map from tablet's host and ports. // It should only be used in discovery and related module. func TabletToMapKey(tablet *topodatapb.Tablet) string { parts := make([]string, 0, 1) for name, port := range tablet.PortMap { parts = append(parts, netutil.JoinHostPort(name, port)) } sort.Strings(parts) parts = append([]string{tablet.Hostname}, parts...) return strings.Join(parts, ",") }
func (client *client) Dial(endPoint *topodatapb.EndPoint, connTimeout time.Duration) error { addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["grpc"]) var err error client.cc, err = grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(connTimeout)) if err != nil { return err } client.c = binlogservicepb.NewUpdateStreamClient(client.cc) return nil }
func addTabletLinks(result *explorerResult, data string) { t := &topo.Tablet{} err := json.Unmarshal([]byte(data), t) if err != nil { return } if port, ok := t.Portmap["vt"]; ok { result.Links["status"] = template.URL(fmt.Sprintf("http://%v/debug/status", netutil.JoinHostPort(t.Hostname, int32(port)))) } }
func (client *client) Dial(endPoint topo.EndPoint, connTimeout time.Duration) error { addr := netutil.JoinHostPort(endPoint.Host, endPoint.NamedPortMap["grpc"]) var err error client.cc, err = grpc.Dial(addr, grpc.WithBlock(), grpc.WithTimeout(connTimeout)) if err != nil { return err } client.c = pbs.NewUpdateStreamClient(client.cc) client.ctx = context.Background() return nil }
func (ex ZkExplorer) addTabletLinks(data string, result *ZkResult) { t := &pb.Tablet{} err := json.Unmarshal([]byte(data), t) if err != nil { return } if port, ok := t.PortMap["vt"]; ok { result.Links["status"] = template.URL(fmt.Sprintf("http://%v/debug/status", netutil.JoinHostPort(t.Hostname, port))) } }
// dial returns a client to use func (client *Client) dial(tablet *topodatapb.Tablet) (*grpc.ClientConn, tabletmanagerservicepb.TabletManagerClient, error) { addr := netutil.JoinHostPort(tablet.Hostname, int32(tablet.PortMap["grpc"])) opt, err := grpcutils.ClientSecureDialOption(*cert, *key, *ca, *name) if err != nil { return nil, nil, err } cc, err := grpc.Dial(addr, opt) if err != nil { return nil, nil, err } return cc, tabletmanagerservicepb.NewTabletManagerClient(cc), nil }
func populateListeningURL() { host, err := netutil.FullyQualifiedHostname() if err != nil { host, err = os.Hostname() if err != nil { log.Fatalf("os.Hostname() failed: %v", err) } } ListeningURL = url.URL{ Scheme: "http", Host: netutil.JoinHostPort(host, int32(*Port)), Path: "/", } }
func main() { defer exit.Recover() defer logutil.Flush() flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s [global parameters] command [command parameters]\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nThe global optional parameters are:\n") flag.PrintDefaults() fmt.Fprintf(os.Stderr, "\nThe commands are listed below. Use '%s <command> -h' for more help.\n\n", os.Args[0]) for _, cmd := range commands { fmt.Fprintf(os.Stderr, " %s", cmd.name) if cmd.params != "" { fmt.Fprintf(os.Stderr, " %s", cmd.params) } fmt.Fprintf(os.Stderr, "\n") } fmt.Fprintf(os.Stderr, "\n") } dbconfigs.RegisterFlags(dbconfigFlags) flag.Parse() tabletAddr = netutil.JoinHostPort("localhost", int32(*port)) action := flag.Arg(0) for _, cmd := range commands { if cmd.name == action { subFlags := flag.NewFlagSet(action, flag.ExitOnError) subFlags.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s %s %s\n\n", os.Args[0], cmd.name, cmd.params) fmt.Fprintf(os.Stderr, "%s\n\n", cmd.help) subFlags.PrintDefaults() } if err := cmd.method(subFlags, flag.Args()[1:]); err != nil { log.Error(err) exit.Return(1) } return } } log.Errorf("invalid action: %v", action) exit.Return(1) }
// dial returns a client to use func (client *Client) dial(ctx context.Context, tablet *topo.TabletInfo) (*grpc.ClientConn, pbs.TabletManagerClient, error) { // create the RPC client, using ctx.Deadline if set, or no timeout. var connectTimeout time.Duration deadline, ok := ctx.Deadline() if ok { connectTimeout = deadline.Sub(time.Now()) if connectTimeout < 0 { return nil, nil, timeoutError{fmt.Errorf("timeout connecting to TabletManager on %v", tablet.Alias)} } } var cc *grpc.ClientConn var err error addr := netutil.JoinHostPort(tablet.Hostname, tablet.Portmap["grpc"]) if connectTimeout == 0 { cc, err = grpc.Dial(addr, grpc.WithBlock()) } else { cc, err = grpc.Dial(addr, grpc.WithBlock(), grpc.WithTimeout(connectTimeout)) } if err != nil { return nil, nil, err } return cc, pbs.NewTabletManagerClient(cc), nil }
// DialTablet creates and initializes gRPCQueryClient. func DialTablet(ctx context.Context, endPoint *pbt.EndPoint, keyspace, shard string, tabletType pbt.TabletType, timeout time.Duration) (tabletconn.TabletConn, error) { // create the RPC client addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["grpc"]) cc, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(timeout)) if err != nil { return nil, err } c := pbs.NewQueryClient(cc) result := &gRPCQueryClient{ endPoint: endPoint, cc: cc, c: c, } if tabletType == pbt.TabletType_UNKNOWN { // we use session gsir, err := c.GetSessionId(ctx, &pb.GetSessionIdRequest{ Keyspace: keyspace, Shard: shard, }) if err != nil { cc.Close() return nil, err } result.sessionID = gsir.SessionId } else { // we use target result.target = &pb.Target{ Keyspace: keyspace, Shard: shard, TabletType: tabletType, } } return result, nil }
func (client *client) Dial(endPoint *pb.EndPoint, connTimeout time.Duration) error { addr := netutil.JoinHostPort(endPoint.Host, endPoint.PortMap["vt"]) var err error client.Client, err = bsonrpc.DialHTTP("tcp", addr, connTimeout) return err }
// MasterAddr returns the host:port address of the master. func (rs *ReplicationStatus) MasterAddr() string { return netutil.JoinHostPort(rs.MasterHost, int32(rs.MasterPort)) }
// Addr returns the fully qualified host name + port for this instance. func (mysqld *Mysqld) Addr() string { hostname := netutil.FullyQualifiedHostnameOrPanic() return netutil.JoinHostPort(hostname, int32(mysqld.config.MysqlPort)) }
func main() { defer exit.Recover() defer logutil.Flush() flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s [global parameters] command [command parameters]\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nThe global optional parameters are:\n") flag.PrintDefaults() fmt.Fprintf(os.Stderr, "\nThe commands are listed below. Use '%s <command> -h' for more help.\n\n", os.Args[0]) for _, cmd := range commands { fmt.Fprintf(os.Stderr, " %s", cmd.name) if cmd.params != "" { fmt.Fprintf(os.Stderr, " %s", cmd.params) } fmt.Fprintf(os.Stderr, "\n") } fmt.Fprintf(os.Stderr, "\n") } flags := dbconfigs.AppConfig | dbconfigs.DbaConfig | dbconfigs.FilteredConfig | dbconfigs.ReplConfig dbconfigs.RegisterFlags(flags) flag.Parse() tabletAddr = netutil.JoinHostPort("localhost", int32(*port)) mycnf := mysqlctl.NewMycnf(uint32(*tabletUID), *mysqlPort) if *mysqlSocket != "" { mycnf.SocketFile = *mysqlSocket } dbcfgs, err := dbconfigs.Init(mycnf.SocketFile, flags) if err != nil { log.Errorf("%v", err) exit.Return(1) } mysqld := mysqlctl.NewMysqld("Dba", "App", mycnf, &dbcfgs.Dba, &dbcfgs.App.ConnParams, &dbcfgs.Repl) defer mysqld.Close() action := flag.Arg(0) for _, cmd := range commands { if cmd.name == action { subFlags := flag.NewFlagSet(action, flag.ExitOnError) subFlags.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s %s %s\n\n", os.Args[0], cmd.name, cmd.params) fmt.Fprintf(os.Stderr, "%s\n\n", cmd.help) subFlags.PrintDefaults() } if err := cmd.method(mysqld, subFlags, flag.Args()[1:]); err != nil { log.Error(err) exit.Return(1) } return } } log.Errorf("invalid action: %v", action) exit.Return(1) }
// TabletAddr returns hostname:vt port associated with a tablet func TabletAddr(tablet *topodatapb.Tablet) string { return netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["vt"]) }
// MysqlAddr returns hostname:mysql port. func (ti *TabletInfo) MysqlAddr() string { return netutil.JoinHostPort(ti.Hostname, int32(ti.PortMap["mysql"])) }
// MysqlIPAddr returns ip:mysql port. func (tablet *Tablet) MysqlIPAddr() string { return netutil.JoinHostPort(tablet.IPAddr, int32(tablet.Portmap["mysql"])) }
// Addr returns hostname:vt port. func (tablet *Tablet) Addr() string { return netutil.JoinHostPort(tablet.Hostname, int32(tablet.Portmap["vt"])) }
// Iteration is a single iteration for the player: get the current status, // try to play, and plays until interrupted, or until an error occurs. func (bpc *BinlogPlayerController) Iteration(ctx context.Context) (err error) { defer func() { if x := recover(); x != nil { log.Errorf("%v: caught panic: %v", bpc, x) err = fmt.Errorf("panic: %v", x) } }() // Apply any special settings necessary for playback of binlogs. // We do it on every iteration to be sure, in case MySQL was restarted. if err := bpc.mysqld.EnableBinlogPlayback(); err != nil { // We failed to apply the required settings, so we shouldn't keep going. return err } // create the db connection, connect it vtClient := binlogplayer.NewDbClient(bpc.dbConfig) if err := vtClient.Connect(); err != nil { return fmt.Errorf("can't connect to database: %v", err) } defer vtClient.Close() // Read the start position startPosition, flags, err := binlogplayer.ReadStartPosition(vtClient, bpc.sourceShard.Uid) if err != nil { return fmt.Errorf("can't read startPosition: %v", err) } // if we shouldn't start, we just error out and try again later if strings.Index(flags, binlogplayer.BlpFlagDontStart) != -1 { return fmt.Errorf("not starting because flag '%v' is set", binlogplayer.BlpFlagDontStart) } // Find the server list for the source shard in our cell addrs, _, err := bpc.ts.GetEndPoints(ctx, bpc.cell, bpc.sourceShard.Keyspace, bpc.sourceShard.Shard, topo.TYPE_REPLICA) if err != nil { return fmt.Errorf("can't find any source tablet for %v %v %v: %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA, err) } if len(addrs.Entries) == 0 { return fmt.Errorf("empty source tablet list for %v %v %v", bpc.cell, bpc.sourceShard.String(), topo.TYPE_REPLICA) } newServerIndex := rand.Intn(len(addrs.Entries)) port, _ := addrs.Entries[newServerIndex].NamedPortMap["vt"] addr := netutil.JoinHostPort(addrs.Entries[newServerIndex].Host, port) // save our current server bpc.playerMutex.Lock() bpc.sourceTablet = topo.TabletAlias{ Cell: bpc.cell, Uid: addrs.Entries[newServerIndex].Uid, } bpc.lastError = nil bpc.playerMutex.Unlock() // check which kind of replication we're doing, tables or keyrange if len(bpc.sourceShard.Tables) > 0 { // tables, first resolve wildcards tables, err := mysqlctl.ResolveTables(bpc.mysqld, bpc.dbName, bpc.sourceShard.Tables) if err != nil { return fmt.Errorf("failed to resolve table names: %v", err) } // tables, just get them player := binlogplayer.NewBinlogPlayerTables(vtClient, addr, tables, startPosition, bpc.stopPosition, bpc.binlogPlayerStats) return player.ApplyBinlogEvents(bpc.interrupted) } // the data we have to replicate is the intersection of the // source keyrange and our keyrange overlap, err := key.KeyRangesOverlap(bpc.sourceShard.KeyRange, bpc.keyRange) if err != nil { return fmt.Errorf("Source shard %v doesn't overlap destination shard %v", bpc.sourceShard.KeyRange, bpc.keyRange) } player := binlogplayer.NewBinlogPlayerKeyRange(vtClient, addr, bpc.keyspaceIDType, overlap, startPosition, bpc.stopPosition, bpc.binlogPlayerStats) return player.ApplyBinlogEvents(bpc.interrupted) }