func commandSplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
online := subFlags.Bool("online", defaultOnline, "do online copy (optional approximate copy, source and destination tablets will not be put out of serving, minimizes downtime during offline copy)")
offline := subFlags.Bool("offline", defaultOffline, "do offline copy (exact copy at a specific GTID, required before shard migration, source and destination tablets will be put out of serving during copy)")
excludeTables := subFlags.String("exclude_tables", "", "comma separated list of tables to exclude")
strategy := subFlags.String("strategy", "", "which strategy to use for restore, use 'vtworker SplitClone --strategy=-help k/s' for more info")
sourceReaderCount := subFlags.Int("source_reader_count", defaultSourceReaderCount, "number of concurrent streaming queries to use on the source")
writeQueryMaxRows := subFlags.Int("write_query_max_rows", defaultWriteQueryMaxRows, "maximum number of rows per write query")
writeQueryMaxSize := subFlags.Int("write_query_max_size", defaultWriteQueryMaxSize, "maximum size (in bytes) per write query")
writeQueryMaxRowsDelete := subFlags.Int("write_query_max_rows_delete", defaultWriteQueryMaxRows, "maximum number of rows per DELETE FROM write query")
minTableSizeForSplit := subFlags.Int("min_table_size_for_split", defaultMinTableSizeForSplit, "tables bigger than this size on disk in bytes will be split into source_reader_count chunks if possible")
destinationWriterCount := subFlags.Int("destination_writer_count", defaultDestinationWriterCount, "number of concurrent RPCs to execute on the destination")
minHealthyRdonlyTablets := subFlags.Int("min_healthy_rdonly_tablets", defaultMinHealthyRdonlyTablets, "minimum number of healthy RDONLY tablets in the source and destination shard at start")
maxTPS := subFlags.Int64("max_tps", defaultMaxTPS, "if non-zero, limit copy to maximum number of (write) transactions/second on the destination (unlimited by default)")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command SplitClone requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
var excludeTableArray []string
if *excludeTables != "" {
excludeTableArray = strings.Split(*excludeTables, ",")
}
worker, err := NewSplitCloneWorker(wr, wi.cell, keyspace, shard, *online, *offline, excludeTableArray, *strategy, *sourceReaderCount, *writeQueryMaxRows, *writeQueryMaxSize, *writeQueryMaxRowsDelete, uint64(*minTableSizeForSplit), *destinationWriterCount, *minHealthyRdonlyTablets, *maxTPS)
if err != nil {
return nil, fmt.Errorf("cannot create split clone worker: %v", err)
}
return worker, nil
}
func commandLegacySplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
excludeTables := subFlags.String("exclude_tables", "", "comma separated list of tables to exclude")
strategy := subFlags.String("strategy", "", "which strategy to use for restore, use 'vtworker LegacySplitClone --strategy=-help k/s' for more info")
sourceReaderCount := subFlags.Int("source_reader_count", defaultSourceReaderCount, "number of concurrent streaming queries to use on the source")
destinationPackCount := subFlags.Int("destination_pack_count", defaultDestinationPackCount, "number of packets to pack in one destination insert")
destinationWriterCount := subFlags.Int("destination_writer_count", defaultDestinationWriterCount, "number of concurrent RPCs to execute on the destination")
minHealthyRdonlyTablets := subFlags.Int("min_healthy_rdonly_tablets", defaultMinHealthyRdonlyTablets, "minimum number of healthy RDONLY tablets before taking out one")
maxTPS := subFlags.Int64("max_tps", defaultMaxTPS, "if non-zero, limit copy to maximum number of (write) transactions/second on the destination (unlimited by default)")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command LegacySplitClone requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
var excludeTableArray []string
if *excludeTables != "" {
excludeTableArray = strings.Split(*excludeTables, ",")
}
worker, err := NewLegacySplitCloneWorker(wr, wi.cell, keyspace, shard, excludeTableArray, *strategy, *sourceReaderCount, *destinationPackCount, *destinationWriterCount, *minHealthyRdonlyTablets, *maxTPS)
if err != nil {
return nil, fmt.Errorf("cannot create split clone worker: %v", err)
}
return worker, nil
}
func commandEmergencyReparentShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
waitSlaveTimeout := subFlags.Duration("wait_slave_timeout", 30*time.Second, "time to wait for slaves to catch up in reparenting")
keyspaceShard := subFlags.String("keyspace_shard", "", "keyspace/shard of the shard that needs to be reparented")
newMaster := subFlags.String("new_master", "", "alias of a tablet that should be the new master")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() == 2 {
// Legacy syntax: "<keyspace/shard> <tablet alias>".
if *newMaster != "" {
return fmt.Errorf("cannot use legacy syntax and flag -new_master for action EmergencyReparentShard at the same time")
}
*keyspaceShard = subFlags.Arg(0)
*newMaster = subFlags.Arg(1)
} else if subFlags.NArg() != 0 {
return fmt.Errorf("action EmergencyReparentShard requires -keyspace_shard=<keyspace/shard> -new_master=<tablet alias>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(*keyspaceShard)
if err != nil {
return err
}
tabletAlias, err := topoproto.ParseTabletAlias(*newMaster)
if err != nil {
return err
}
return wr.EmergencyReparentShard(ctx, keyspace, shard, tabletAlias, *waitSlaveTimeout)
}
func commandVerticalSplitClone(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
tables := subFlags.String("tables", "", "comma separated list of tables to replicate (used for vertical split)")
strategy := subFlags.String("strategy", "", "which strategy to use for restore, use 'vtworker VerticalSplitClone --strategy=-help k/s' for more info")
sourceReaderCount := subFlags.Int("source_reader_count", defaultSourceReaderCount, "number of concurrent streaming queries to use on the source")
destinationPackCount := subFlags.Int("destination_pack_count", defaultDestinationPackCount, "number of packets to pack in one destination insert")
minTableSizeForSplit := subFlags.Int("min_table_size_for_split", defaultMinTableSizeForSplit, "tables bigger than this size on disk in bytes will be split into source_reader_count chunks if possible")
destinationWriterCount := subFlags.Int("destination_writer_count", defaultDestinationWriterCount, "number of concurrent RPCs to execute on the destination")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command VerticalSplitClone requires <destination keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
var tableArray []string
if *tables != "" {
tableArray = strings.Split(*tables, ",")
}
worker, err := NewVerticalSplitCloneWorker(wr, wi.cell, keyspace, shard, tableArray, *strategy, *sourceReaderCount, *destinationPackCount, uint64(*minTableSizeForSplit), *destinationWriterCount)
if err != nil {
return nil, fmt.Errorf("cannot create worker: %v", err)
}
return worker, nil
}
func commandListBackups(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 1 {
return fmt.Errorf("action ListBackups requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return err
}
bucket := fmt.Sprintf("%v/%v", keyspace, shard)
bs, err := backupstorage.GetBackupStorage()
if err != nil {
return err
}
defer bs.Close()
bhs, err := bs.ListBackups(bucket)
if err != nil {
return err
}
for _, bh := range bhs {
wr.Logger().Printf("%v\n", bh.Name())
}
return nil
}
func commandVerticalSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command VerticalSplitDiff requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
return NewVerticalSplitDiffWorker(wr, wi.cell, keyspace, shard), nil
}
func commandVerticalSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
minHealthyRdonlyTablets := subFlags.Int("min_healthy_rdonly_tablets", defaultMinHealthyRdonlyTablets, "minimum number of healthy RDONLY tablets before taking out one")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command VerticalSplitDiff requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
return NewVerticalSplitDiffWorker(wr, wi.cell, keyspace, shard, *minHealthyRdonlyTablets), nil
}
func commandInitShardMaster(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
force := subFlags.Bool("force", false, "will force the reparent even if the provided tablet is not a master or the shard master")
waitSlaveTimeout := subFlags.Duration("wait_slave_timeout", 30*time.Second, "time to wait for slaves to catch up in reparenting")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 2 {
return fmt.Errorf("action InitShardMaster requires <keyspace/shard> <tablet alias>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return err
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1))
if err != nil {
return err
}
return wr.InitShardMaster(ctx, keyspace, shard, tabletAlias, *force, *waitSlaveTimeout)
}
func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
excludeTables := subFlags.String("exclude_tables", "", "comma separated list of tables to exclude")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command SplitDiff requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
var excludeTableArray []string
if *excludeTables != "" {
excludeTableArray = strings.Split(*excludeTables, ",")
}
return NewSplitDiffWorker(wr, wi.cell, keyspace, shard, excludeTableArray), nil
}
func commandEmergencyReparentShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
waitSlaveTimeout := subFlags.Duration("wait_slave_timeout", 30*time.Second, "time to wait for slaves to catch up in reparenting")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 2 {
return fmt.Errorf("action EmergencyReparentShard requires <keyspace/shard> <tablet alias>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return err
}
tabletAlias, err := topoproto.ParseTabletAlias(subFlags.Arg(1))
if err != nil {
return err
}
return wr.EmergencyReparentShard(ctx, keyspace, shard, tabletAlias, *waitSlaveTimeout)
}
func newShardCache(ts topo.Server) *VersionedObjectCacheMap {
return NewVersionedObjectCacheMap(func(key string) *VersionedObjectCache {
return NewVersionedObjectCache(func(ctx context.Context) (VersionedObject, error) {
keyspace, shard, err := topoproto.ParseKeyspaceShard(key)
if err != nil {
return nil, err
}
s, err := ts.GetShard(ctx, keyspace, shard)
if err != nil {
return nil, err
}
return &Shard{
KeyspaceName: s.Keyspace(),
ShardName: s.ShardName(),
Shard: s.Shard,
}, nil
})
})
}
func commandRemoveBackup(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() != 2 {
return fmt.Errorf("action RemoveBackup requires <keyspace/shard> <backup name>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return err
}
bucket := fmt.Sprintf("%v/%v", keyspace, shard)
name := subFlags.Arg(1)
bs, err := backupstorage.GetBackupStorage()
if err != nil {
return err
}
return bs.RemoveBackup(bucket, name)
}
func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
sourceUID := subFlags.Int("source_uid", 0, "uid of the source shard to run the diff against")
excludeTables := subFlags.String("exclude_tables", "", "comma separated list of tables to exclude")
minHealthyRdonlyTablets := subFlags.Int("min_healthy_rdonly_tablets", defaultMinHealthyRdonlyTablets, "minimum number of healthy RDONLY tablets before taking out one")
if err := subFlags.Parse(args); err != nil {
return nil, err
}
if subFlags.NArg() != 1 {
subFlags.Usage()
return nil, fmt.Errorf("command SplitDiff requires <keyspace/shard>")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
if err != nil {
return nil, err
}
var excludeTableArray []string
if *excludeTables != "" {
excludeTableArray = strings.Split(*excludeTables, ",")
}
return NewSplitDiffWorker(wr, wi.cell, keyspace, shard, uint32(*sourceUID), excludeTableArray, *minHealthyRdonlyTablets), nil
}
func commandPlannedReparentShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
waitSlaveTimeout := subFlags.Duration("wait_slave_timeout", 30*time.Second, "time to wait for slaves to catch up in reparenting")
keyspaceShard := subFlags.String("keyspace_shard", "", "keyspace/shard of the shard that needs to be reparented")
newMaster := subFlags.String("new_master", "", "alias of a tablet that should be the new master")
avoidMaster := subFlags.String("avoid_master", "", "alias of a tablet that should not be the master, i.e. reparent to any other tablet if this one is the master")
if err := subFlags.Parse(args); err != nil {
return err
}
if subFlags.NArg() == 2 {
// Legacy syntax: "<keyspace/shard> <tablet alias>".
if *keyspaceShard != "" || *newMaster != "" {
return fmt.Errorf("cannot use legacy syntax and flags -keyspace_shard and -new_master for action PlannedReparentShard at the same time")
}
*keyspaceShard = subFlags.Arg(0)
*newMaster = subFlags.Arg(1)
} else if subFlags.NArg() != 0 {
return fmt.Errorf("action PlannedReparentShard requires -keyspace_shard=<keyspace/shard> [-new_master=<tablet alias>] [-avoid_master=<tablet alias>]")
}
keyspace, shard, err := topoproto.ParseKeyspaceShard(*keyspaceShard)
if err != nil {
return err
}
var newMasterAlias, avoidMasterAlias *topodatapb.TabletAlias
if *newMaster != "" {
newMasterAlias, err = topoproto.ParseTabletAlias(*newMaster)
if err != nil {
return err
}
}
if *avoidMaster != "" {
avoidMasterAlias, err = topoproto.ParseTabletAlias(*avoidMaster)
if err != nil {
return err
}
}
return wr.PlannedReparentShard(ctx, keyspace, shard, newMasterAlias, avoidMasterAlias, *waitSlaveTimeout)
}
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository, realtimeStats *realtimeStats) {
tabletHealthCache := newTabletHealthCache(ts)
tmClient := tmclient.NewTabletManagerClient()
// Cells
handleCollection("cells", func(r *http.Request) (interface{}, error) {
if getItemPath(r.URL.Path) != "" {
return nil, errors.New("cells can only be listed, not retrieved")
}
return ts.GetKnownCells(ctx)
})
// Keyspaces
handleCollection("keyspaces", func(r *http.Request) (interface{}, error) {
keyspace := getItemPath(r.URL.Path)
switch r.Method {
case "GET":
// List all keyspaces.
if keyspace == "" {
return ts.GetKeyspaces(ctx)
}
// Get the keyspace record.
k, err := ts.GetKeyspace(ctx, keyspace)
// Pass the embedded proto directly or jsonpb will panic.
return k.Keyspace, err
// Perform an action on a keyspace.
case "POST":
if keyspace == "" {
return nil, errors.New("A POST request needs a keyspace in the URL")
}
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("A POST request must specify action")
}
return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil
default:
return nil, fmt.Errorf("unsupported HTTP method: %v", r.Method)
}
})
// Shards
handleCollection("shards", func(r *http.Request) (interface{}, error) {
shardPath := getItemPath(r.URL.Path)
if !strings.Contains(shardPath, "/") {
return nil, fmt.Errorf("invalid shard path: %q", shardPath)
}
parts := strings.SplitN(shardPath, "/", 2)
keyspace := parts[0]
shard := parts[1]
// List the shards in a keyspace.
if shard == "" {
return ts.GetShardNames(ctx, keyspace)
}
// Perform an action on a shard.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil
}
// Get the shard record.
si, err := ts.GetShard(ctx, keyspace, shard)
// Pass the embedded proto directly or jsonpb will panic.
return si.Shard, err
})
// SrvKeyspace
handleCollection("srv_keyspace", func(r *http.Request) (interface{}, error) {
keyspacePath := getItemPath(r.URL.Path)
parts := strings.SplitN(keyspacePath, "/", 2)
// Request was incorrectly formatted.
if len(parts) != 2 {
return nil, fmt.Errorf("invalid srvkeyspace path: %q expected path: /srv_keyspace/<cell>/<keyspace>", keyspacePath)
}
cell := parts[0]
keyspace := parts[1]
if cell == "local" {
if *localCell == "" {
return nil, fmt.Errorf("local cell requested, but not specified. Please set with -cell flag")
}
cell = *localCell
}
// If a keyspace is provided then return the specified srvkeyspace.
if keyspace != "" {
srvKeyspace, err := ts.GetSrvKeyspace(ctx, cell, keyspace)
if err != nil {
return nil, fmt.Errorf("Can't get server keyspace: %v", err)
}
return srvKeyspace, nil
}
// Else return the srvKeyspace from all keyspaces.
srvKeyspaces := make(map[string]interface{})
keyspaceNamesList, err := ts.GetSrvKeyspaceNames(ctx, cell)
if err != nil {
return nil, fmt.Errorf("can't get list of SrvKeyspaceNames for cell %q: GetSrvKeyspaceNames returned: %v", cell, err)
}
for _, keyspaceName := range keyspaceNamesList {
err := addSrvkeyspace(ctx, ts, cell, keyspaceName, srvKeyspaces)
if err != nil {
return nil, err
}
}
return srvKeyspaces, nil
})
// Tablets
handleCollection("tablets", func(r *http.Request) (interface{}, error) {
tabletPath := getItemPath(r.URL.Path)
// List tablets based on query params.
if tabletPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
shardRef := r.FormValue("shard")
cell := r.FormValue("cell")
if shardRef != "" {
// Look up by keyspace/shard, and optionally cell.
keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef)
if err != nil {
return nil, err
}
if cell != "" {
result, err := ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell})
if err != nil && err != topo.ErrPartialResult {
return result, err
}
return result, nil
}
result, err := ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
if err != nil && err != topo.ErrPartialResult {
return result, err
}
return result, nil
}
// Get all tablets in a cell.
if cell == "" {
return nil, errors.New("cell param required")
}
return ts.GetTabletsByCell(ctx, cell)
}
// Get tablet health.
if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" {
tabletAlias, err := topoproto.ParseTabletAlias(parts[0])
if err != nil {
return nil, err
}
return tabletHealthCache.Get(ctx, tabletAlias)
}
tabletAlias, err := topoproto.ParseTabletAlias(tabletPath)
if err != nil {
return nil, err
}
// Perform an action on a tablet.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil
}
// Get the tablet record.
t, err := ts.GetTablet(ctx, tabletAlias)
// Pass the embedded proto directly or jsonpb will panic.
return t.Tablet, err
})
// Healthcheck real time status per (cell, keyspace, tablet type, metric).
handleCollection("tablet_statuses", func(r *http.Request) (interface{}, error) {
targetPath := getItemPath(r.URL.Path)
// Get the heatmap data based on query parameters.
if targetPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
keyspace := r.FormValue("keyspace")
cell := r.FormValue("cell")
tabletType := r.FormValue("type")
_, err := topoproto.ParseTabletType(tabletType)
// Excluding the case where parse fails because all tabletTypes was chosen.
if err != nil && tabletType != "all" {
return nil, fmt.Errorf("invalid tablet type: %v ", err)
}
metric := r.FormValue("metric")
// Setting default values if none was specified in the query params.
if keyspace == "" {
keyspace = "all"
}
if cell == "" {
cell = "all"
}
if tabletType == "" {
tabletType = "all"
}
if metric == "" {
metric = "health"
}
if realtimeStats == nil {
return nil, fmt.Errorf("realtimeStats not initialized")
}
heatmap, err := realtimeStats.heatmapData(keyspace, cell, tabletType, metric)
if err != nil {
return nil, fmt.Errorf("couldn't get heatmap data: %v", err)
}
return heatmap, nil
}
return nil, fmt.Errorf("invalid target path: %q expected path: ?keyspace=<keyspace>&cell=<cell>&type=<type>&metric=<metric>", targetPath)
})
handleCollection("tablet_health", func(r *http.Request) (interface{}, error) {
tabletPath := getItemPath(r.URL.Path)
parts := strings.SplitN(tabletPath, "/", 2)
// Request was incorrectly formatted.
if len(parts) != 2 {
return nil, fmt.Errorf("invalid tablet_health path: %q expected path: /tablet_health/<cell>/<uid>", tabletPath)
}
if realtimeStats == nil {
return nil, fmt.Errorf("realtimeStats not initialized")
}
cell := parts[0]
uidStr := parts[1]
uid, err := topoproto.ParseUID(uidStr)
if err != nil {
return nil, fmt.Errorf("incorrect uid: %v", err)
}
tabletAlias := topodatapb.TabletAlias{
Cell: cell,
Uid: uid,
}
tabletStat, err := realtimeStats.tabletStats(&tabletAlias)
if err != nil {
return nil, fmt.Errorf("could not get tabletStats: %v", err)
}
return tabletStat, nil
})
handleCollection("topology_info", func(r *http.Request) (interface{}, error) {
targetPath := getItemPath(r.URL.Path)
// Retrieving topology information (keyspaces, cells, and types) based on query params.
if targetPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
keyspace := r.FormValue("keyspace")
cell := r.FormValue("cell")
// Setting default values if none was specified in the query params.
if keyspace == "" {
keyspace = "all"
}
if cell == "" {
cell = "all"
}
if realtimeStats == nil {
return nil, fmt.Errorf("realtimeStats not initialized")
}
return realtimeStats.topologyInfo(keyspace, cell), nil
}
return nil, fmt.Errorf("invalid target path: %q expected path: ?keyspace=<keyspace>&cell=<cell>", targetPath)
})
// Vtctl Command
http.HandleFunc(apiPrefix+"vtctl/", func(w http.ResponseWriter, r *http.Request) {
if err := acl.CheckAccessHTTP(r, acl.ADMIN); err != nil {
httpErrorf(w, r, "Access denied")
return
}
var args []string
resp := struct {
Error string
Output string
}{}
if err := unmarshalRequest(r, &args); err != nil {
httpErrorf(w, r, "can't unmarshal request: %v", err)
return
}
logstream := logutil.NewMemoryLogger()
wr := wrangler.New(logstream, ts, tmClient)
// TODO(enisoc): Context for run command should be request-scoped.
err := vtctl.RunCommand(ctx, wr, args)
if err != nil {
resp.Error = err.Error()
}
resp.Output = logstream.String()
data, err := json.MarshalIndent(resp, "", " ")
if err != nil {
httpErrorf(w, r, "json error: %v", err)
return
}
w.Header().Set("Content-Type", jsonContentType)
w.Write(data)
})
// Schema Change
http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) {
if err := acl.CheckAccessHTTP(r, acl.ADMIN); err != nil {
httpErrorf(w, r, "Access denied")
return
}
req := struct {
Keyspace, SQL string
SlaveTimeoutSeconds int
}{}
if err := unmarshalRequest(r, &req); err != nil {
httpErrorf(w, r, "can't unmarshal request: %v", err)
return
}
if req.SlaveTimeoutSeconds <= 0 {
req.SlaveTimeoutSeconds = 10
}
logger := logutil.NewCallbackLogger(func(ev *logutilpb.Event) {
w.Write([]byte(logutil.EventString(ev)))
})
wr := wrangler.New(logger, ts, tmClient)
executor := schemamanager.NewTabletExecutor(
wr, time.Duration(req.SlaveTimeoutSeconds)*time.Second)
schemamanager.Run(ctx,
schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor)
})
}
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) {
tabletHealthCache := newTabletHealthCache(ts)
// Cells
handleCollection("cells", func(r *http.Request) (interface{}, error) {
if getItemPath(r.URL.Path) != "" {
return nil, errors.New("cells can only be listed, not retrieved")
}
return ts.GetKnownCells(ctx)
})
// Keyspaces
handleCollection("keyspaces", func(r *http.Request) (interface{}, error) {
keyspace := getItemPath(r.URL.Path)
// List all keyspaces.
if keyspace == "" {
return ts.GetKeyspaces(ctx)
}
// Perform an action on a keyspace.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil
}
// Get the keyspace record.
return ts.GetKeyspace(ctx, keyspace)
})
// Shards
handleCollection("shards", func(r *http.Request) (interface{}, error) {
shardPath := getItemPath(r.URL.Path)
if !strings.Contains(shardPath, "/") {
return nil, fmt.Errorf("invalid shard path: %q", shardPath)
}
parts := strings.SplitN(shardPath, "/", 2)
keyspace := parts[0]
shard := parts[1]
// List the shards in a keyspace.
if shard == "" {
return ts.GetShardNames(ctx, keyspace)
}
// Perform an action on a shard.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil
}
// Get the shard record.
return ts.GetShard(ctx, keyspace, shard)
})
// Tablets
handleCollection("tablets", func(r *http.Request) (interface{}, error) {
tabletPath := getItemPath(r.URL.Path)
// List tablets based on query params.
if tabletPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
shardRef := r.FormValue("shard")
cell := r.FormValue("cell")
if shardRef != "" {
// Look up by keyspace/shard, and optionally cell.
keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef)
if err != nil {
return nil, err
}
if cell != "" {
return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell})
}
return ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
}
// Get all tablets in a cell.
if cell == "" {
return nil, errors.New("cell param required")
}
return ts.GetTabletsByCell(ctx, cell)
}
// Get tablet health.
if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" {
tabletAlias, err := topoproto.ParseTabletAlias(parts[0])
if err != nil {
return nil, err
}
return tabletHealthCache.Get(ctx, tabletAlias)
}
tabletAlias, err := topoproto.ParseTabletAlias(tabletPath)
if err != nil {
return nil, err
}
// Perform an action on a tablet.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil
}
// Get the tablet record.
return ts.GetTablet(ctx, tabletAlias)
})
// EndPoints
handleCollection("endpoints", func(r *http.Request) (interface{}, error) {
// We expect cell/keyspace/shard/tabletType.
epPath := getItemPath(r.URL.Path)
parts := strings.Split(epPath, "/")
if len(parts) != 4 {
return nil, fmt.Errorf("invalid cell/keyspace/shard/tabletType: %q", epPath)
}
if parts[3] == "" {
// tabletType is empty, so list the tablet types.
return ts.GetSrvTabletTypesPerShard(ctx, parts[0], parts[1], parts[2])
}
tabletType, err := topoproto.ParseTabletType(parts[3])
if err != nil {
return nil, fmt.Errorf("invalid tablet type %v: %v", parts[3], err)
}
// Get the endpoints object for a specific type.
ep, _, err := ts.GetEndPoints(ctx, parts[0], parts[1], parts[2], tabletType)
return ep, err
})
// Schema Change
http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) {
req := struct{ Keyspace, SQL string }{}
if err := unmarshalRequest(r, &req); err != nil {
httpErrorf(w, r, "can't unmarshal request: %v", err)
return
}
executor := schemamanager.NewTabletExecutor(
tmclient.NewTabletManagerClient(),
ts)
schemamanager.Run(ctx,
schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor)
})
// VSchema
http.HandleFunc(apiPrefix+"vschema/", func(w http.ResponseWriter, r *http.Request) {
// Save VSchema
if r.Method == "POST" {
vschema, err := ioutil.ReadAll(r.Body)
if err != nil {
httpErrorf(w, r, "can't read request body: %v", err)
return
}
if err := ts.SaveVSchema(ctx, string(vschema)); err != nil {
httpErrorf(w, r, "can't save vschema: %v", err)
}
return
}
// Get VSchema
vschema, err := ts.GetVSchema(ctx)
if err != nil {
httpErrorf(w, r, "can't get vschema: %v", err)
return
}
w.Header().Set("Content-Type", jsonContentType)
w.Write([]byte(vschema))
})
}
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) {
tabletHealthCache := newTabletHealthCache(ts)
// Cells
handleCollection("cells", func(r *http.Request) (interface{}, error) {
if getItemPath(r.URL.Path) != "" {
return nil, errors.New("cells can only be listed, not retrieved")
}
return ts.GetKnownCells(ctx)
})
// Keyspaces
handleCollection("keyspaces", func(r *http.Request) (interface{}, error) {
keyspace := getItemPath(r.URL.Path)
// List all keyspaces.
if keyspace == "" {
return ts.GetKeyspaces(ctx)
}
// Perform an action on a keyspace.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil
}
// Get the keyspace record.
return ts.GetKeyspace(ctx, keyspace)
})
// Shards
handleCollection("shards", func(r *http.Request) (interface{}, error) {
shardPath := getItemPath(r.URL.Path)
if !strings.Contains(shardPath, "/") {
return nil, fmt.Errorf("invalid shard path: %q", shardPath)
}
parts := strings.SplitN(shardPath, "/", 2)
keyspace := parts[0]
shard := parts[1]
// List the shards in a keyspace.
if shard == "" {
return ts.GetShardNames(ctx, keyspace)
}
// Perform an action on a shard.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil
}
// Get the shard record.
return ts.GetShard(ctx, keyspace, shard)
})
// Tablets
handleCollection("tablets", func(r *http.Request) (interface{}, error) {
tabletPath := getItemPath(r.URL.Path)
// List tablets based on query params.
if tabletPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
shardRef := r.FormValue("shard")
cell := r.FormValue("cell")
if shardRef != "" {
// Look up by keyspace/shard, and optionally cell.
keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef)
if err != nil {
return nil, err
}
if cell != "" {
return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell})
}
return ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
}
// Get all tablets in a cell.
if cell == "" {
return nil, errors.New("cell param required")
}
return ts.GetTabletsByCell(ctx, cell)
}
// Get tablet health.
if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" {
tabletAlias, err := topoproto.ParseTabletAlias(parts[0])
if err != nil {
return nil, err
}
return tabletHealthCache.Get(ctx, tabletAlias)
}
tabletAlias, err := topoproto.ParseTabletAlias(tabletPath)
if err != nil {
return nil, err
}
// Perform an action on a tablet.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil
}
// Get the tablet record.
return ts.GetTablet(ctx, tabletAlias)
})
// Schema Change
http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) {
req := struct{ Keyspace, SQL string }{}
if err := unmarshalRequest(r, &req); err != nil {
httpErrorf(w, r, "can't unmarshal request: %v", err)
return
}
executor := schemamanager.NewTabletExecutor(
tmclient.NewTabletManagerClient(),
ts)
schemamanager.Run(ctx,
schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor)
})
}
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) {
tabletHealthCache := newTabletHealthCache(ts)
tmClient := tmclient.NewTabletManagerClient()
// Cells
handleCollection("cells", func(r *http.Request) (interface{}, error) {
if getItemPath(r.URL.Path) != "" {
return nil, errors.New("cells can only be listed, not retrieved")
}
return ts.GetKnownCells(ctx)
})
// Keyspaces
handleCollection("keyspaces", func(r *http.Request) (interface{}, error) {
keyspace := getItemPath(r.URL.Path)
// List all keyspaces.
if keyspace == "" {
return ts.GetKeyspaces(ctx)
}
// Perform an action on a keyspace.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil
}
// Get the keyspace record.
return ts.GetKeyspace(ctx, keyspace)
})
// Shards
handleCollection("shards", func(r *http.Request) (interface{}, error) {
shardPath := getItemPath(r.URL.Path)
if !strings.Contains(shardPath, "/") {
return nil, fmt.Errorf("invalid shard path: %q", shardPath)
}
parts := strings.SplitN(shardPath, "/", 2)
keyspace := parts[0]
shard := parts[1]
// List the shards in a keyspace.
if shard == "" {
return ts.GetShardNames(ctx, keyspace)
}
// Perform an action on a shard.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil
}
// Get the shard record.
return ts.GetShard(ctx, keyspace, shard)
})
//SrvKeyspace
handleCollection("srv_keyspace", func(r *http.Request) (interface{}, error) {
keyspacePath := getItemPath(r.URL.Path)
parts := strings.SplitN(keyspacePath, "/", 2)
//request was incorrectly formatted
if len(parts) != 2 {
return nil, fmt.Errorf("invalid srvkeyspace path: %q expected path: /srv_keyspace/<cell>/<keyspace>", keyspacePath)
}
cell := parts[0]
keyspace := parts[1]
if cell == "local" {
if *localCell == "" {
return nil, fmt.Errorf("local cell requested, but not specified. Please set with -cell flag")
}
cell = *localCell
}
//If a keyspace is provided then return the specified srvkeyspace
if keyspace != "" {
srvKeyspace, err := ts.GetSrvKeyspace(ctx, cell, keyspace)
if err != nil {
return nil, fmt.Errorf("Can't get server keyspace: %v", err)
}
return srvKeyspace, nil
}
//Else return the srvKeyspace from all keyspaces
srvKeyspaces := make(map[string]interface{})
keyspaceNamesList, err := ts.GetSrvKeyspaceNames(ctx, cell)
if err != nil {
return nil, fmt.Errorf("can't get list of SrvKeyspaceNames for cell %q: GetSrvKeyspaceNames returned: %v", cell, err)
}
for _, keyspaceName := range keyspaceNamesList {
err := addSrvkeyspace(ctx, ts, cell, keyspaceName, srvKeyspaces)
if err != nil {
return nil, err
}
}
return srvKeyspaces, nil
})
// Tablets
handleCollection("tablets", func(r *http.Request) (interface{}, error) {
tabletPath := getItemPath(r.URL.Path)
// List tablets based on query params.
if tabletPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
shardRef := r.FormValue("shard")
cell := r.FormValue("cell")
if shardRef != "" {
// Look up by keyspace/shard, and optionally cell.
keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef)
if err != nil {
return nil, err
}
if cell != "" {
return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell})
}
return ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
}
// Get all tablets in a cell.
if cell == "" {
return nil, errors.New("cell param required")
}
return ts.GetTabletsByCell(ctx, cell)
}
// Get tablet health.
if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" {
tabletAlias, err := topoproto.ParseTabletAlias(parts[0])
if err != nil {
return nil, err
}
return tabletHealthCache.Get(ctx, tabletAlias)
}
tabletAlias, err := topoproto.ParseTabletAlias(tabletPath)
if err != nil {
return nil, err
}
// Perform an action on a tablet.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil
}
// Get the tablet record.
return ts.GetTablet(ctx, tabletAlias)
})
// Schema Change
http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) {
req := struct {
Keyspace, SQL string
SlaveTimeoutSeconds int
}{}
if err := unmarshalRequest(r, &req); err != nil {
httpErrorf(w, r, "can't unmarshal request: %v", err)
return
}
if req.SlaveTimeoutSeconds <= 0 {
req.SlaveTimeoutSeconds = 10
}
logger := logutil.NewCallbackLogger(func(ev *logutilpb.Event) {
w.Write([]byte(logutil.EventString(ev)))
})
wr := wrangler.New(logger, ts, tmClient)
executor := schemamanager.NewTabletExecutor(
wr, time.Duration(req.SlaveTimeoutSeconds)*time.Second)
schemamanager.Run(ctx,
schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor)
})
}
