// NewQueryResultReaderForTablet creates a new QueryResultReader for // the provided tablet / sql query func NewQueryResultReaderForTablet(ctx context.Context, ts topo.Server, tabletAlias topo.TabletAlias, sql string) (*QueryResultReader, error) { tablet, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return nil, err } endPoint, err := tablet.EndPoint() if err != nil { return nil, err } conn, err := tabletconn.GetDialer()(ctx, *endPoint, tablet.Keyspace, tablet.Shard, *remoteActionsTimeout) if err != nil { return nil, err } sr, clientErrFn, err := conn.StreamExecute(ctx, sql, make(map[string]interface{}), 0) if err != nil { return nil, err } // read the columns, or grab the error cols, ok := <-sr if !ok { return nil, fmt.Errorf("Cannot read Fields for query '%v': %v", sql, clientErrFn()) } return &QueryResultReader{ Output: sr, Fields: cols.Fields, conn: conn, clientErrFn: clientErrFn, }, nil }
// If error is not nil, the results in the dictionary are incomplete. func GetTabletMap(ts topo.Server, tabletAliases []topo.TabletAlias) (map[topo.TabletAlias]*topo.TabletInfo, error) { wg := sync.WaitGroup{} mutex := sync.Mutex{} tabletMap := make(map[topo.TabletAlias]*topo.TabletInfo) var someError error for _, tabletAlias := range tabletAliases { wg.Add(1) go func(tabletAlias topo.TabletAlias) { defer wg.Done() tabletInfo, err := ts.GetTablet(tabletAlias) mutex.Lock() if err != nil { relog.Warning("%v: %v", tabletAlias, err) // There can be data races removing nodes - ignore them for now. if err != topo.ErrNoNode { someError = err } } else { tabletMap[tabletAlias] = tabletInfo } mutex.Unlock() }(tabletAlias) } wg.Wait() return tabletMap, someError }
// NewQueryResultReaderForTablet creates a new QueryResultReader for // the provided tablet / sql query func NewQueryResultReaderForTablet(ctx context.Context, ts topo.Server, tabletAlias *topodatapb.TabletAlias, sql string) (*QueryResultReader, error) { shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) tablet, err := ts.GetTablet(shortCtx, tabletAlias) cancel() if err != nil { return nil, err } conn, err := tabletconn.GetDialer()(tablet.Tablet, *remoteActionsTimeout) if err != nil { return nil, err } stream, err := conn.StreamExecute(ctx, &querypb.Target{ Keyspace: tablet.Tablet.Keyspace, Shard: tablet.Tablet.Shard, TabletType: tablet.Tablet.Type, }, sql, make(map[string]interface{}), nil) if err != nil { return nil, err } // read the columns, or grab the error cols, err := stream.Recv() if err != nil { return nil, fmt.Errorf("Cannot read Fields for query '%v': %v", sql, err) } return &QueryResultReader{ output: stream, fields: cols.Fields, conn: conn, }, nil }
// TabletExternallyReparented updates all topo records so the current // tablet is the new master for this shard. It is called by the RPC // server. func TabletExternallyReparented(ts topo.Server, tabletAlias topo.TabletAlias, actionTimeout, lockTimeout time.Duration) error { // we're apprently not the master yet, so let's do the work tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } // fast quick check on the shard shardInfo, err := ts.GetShard(tablet.Keyspace, tablet.Shard) if err != nil { return err } if shardInfo.MasterAlias == tabletAlias { return nil } // grab the shard lock actionNode := actionnode.ShardExternallyReparented(tabletAlias) interrupted := make(chan struct{}) lockPath, err := actionNode.LockShard(ts, tablet.Keyspace, tablet.Shard, lockTimeout, interrupted) if err != nil { return err } // do the work err = tabletExternallyReparentedLocked(ts, tablet, actionTimeout, lockTimeout, interrupted) // release the lock in any case return actionNode.UnlockShard(ts, tablet.Keyspace, tablet.Shard, lockPath, err) }
func SlaveWasRestarted(ts topo.Server, tabletAlias topo.TabletAlias, swrd *actionnode.SlaveWasRestartedArgs) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } // Once this action completes, update authoritive tablet node first. tablet.Parent = swrd.Parent if tablet.Type == topo.TYPE_MASTER { tablet.Type = topo.TYPE_SPARE tablet.State = topo.STATE_READ_ONLY } err = topo.UpdateTablet(ts, tablet) if err != nil { return err } // Update the new tablet location in the replication graph now that // we've updated the tablet. err = topo.CreateTabletReplicationData(ts, tablet.Tablet) if err != nil && err != topo.ErrNodeExists { return err } return nil }
// CopyTablets will create the tablets in the destination topo func CopyTablets(fromTS, toTS topo.Server) { cells, err := fromTS.GetKnownCells() if err != nil { log.Fatalf("fromTS.GetKnownCells failed: %v", err) } wg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} for _, cell := range cells { wg.Add(1) go func(cell string) { defer wg.Done() tabletAliases, err := fromTS.GetTabletsByCell(cell) if err != nil { rec.RecordError(err) } else { for _, tabletAlias := range tabletAliases { wg.Add(1) go func(tabletAlias topo.TabletAlias) { defer wg.Done() // read the source tablet ti, err := fromTS.GetTablet(tabletAlias) if err != nil { rec.RecordError(err) return } // try to create the destination err = toTS.CreateTablet(ti.Tablet) if err == topo.ErrNodeExists { // update the destination tablet log.Warningf("tablet %v already exists, updating it", tabletAlias) err = toTS.UpdateTabletFields(ti.Alias(), func(t *topo.Tablet) error { *t = *ti.Tablet return nil }) } if err != nil { rec.RecordError(err) return } // create the replication paths // for masters only here if ti.Type == topo.TYPE_MASTER { if err = toTS.CreateReplicationPath(ti.Keyspace, ti.Shard, ti.Alias().String()); err != nil && err != topo.ErrNodeExists { rec.RecordError(err) } } }(tabletAlias) } } }(cell) } wg.Wait() if rec.HasErrors() { log.Fatalf("copyTablets failed: %v", rec.Error()) } }
// NewRestartableResultReader creates a new RestartableResultReader for // the provided tablet and chunk. // It will automatically create the necessary query to read all rows within // the chunk. // NOTE: We assume that the Columns field in "td" was ordered by a preceding // call to reorderColumnsPrimaryKeyFirst(). func NewRestartableResultReader(ctx context.Context, logger logutil.Logger, ts topo.Server, tabletAlias *topodatapb.TabletAlias, td *tabletmanagerdatapb.TableDefinition, chunk chunk) (*RestartableResultReader, error) { shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout) tablet, err := ts.GetTablet(shortCtx, tabletAlias) cancel() if err != nil { return nil, fmt.Errorf("tablet=%v table=%v chunk=%v: Failed to resolve tablet alias: %v", topoproto.TabletAliasString(tabletAlias), td.Name, chunk, err) } conn, err := tabletconn.GetDialer()(tablet.Tablet, *remoteActionsTimeout) if err != nil { return nil, fmt.Errorf("tablet=%v table=%v chunk=%v: Failed to get dialer for tablet: %v", topoproto.TabletAliasString(tabletAlias), td.Name, chunk, err) } r := &RestartableResultReader{ ctx: ctx, logger: logger, tablet: tablet.Tablet, td: td, chunk: chunk, conn: conn, } if err := r.startStream(); err != nil { return nil, err } logger.Infof("tablet=%v table=%v chunk=%v: Starting to stream rows using query '%v'.", topoproto.TabletAliasString(tabletAlias), td.Name, chunk, r.query) return r, nil }
// ChangeType changes the type of the tablet and possibly also updates // the health informaton for it. Make this external, since these // transitions need to be forced from time to time. // // - if health is nil, we don't touch the Tablet's Health record. // - if health is an empty map, we clear the Tablet's Health record. // - if health has values, we overwrite the Tablet's Health record. func ChangeType(ctx context.Context, ts topo.Server, tabletAlias topo.TabletAlias, newType topo.TabletType, health map[string]string) error { tablet, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return err } if !topo.IsTrivialTypeChange(tablet.Type, newType) { return fmt.Errorf("cannot change tablet type %v -> %v %v", tablet.Type, newType, tabletAlias) } tablet.Type = newType if newType == topo.TYPE_IDLE { tablet.Keyspace = "" tablet.Shard = "" tablet.KeyRange = key.KeyRange{} tablet.Health = health } if health != nil { if len(health) == 0 { tablet.Health = nil } else { tablet.Health = health } } return topo.UpdateTablet(ctx, ts, tablet) }
func SlaveWasPromoted(ts topo.Server, tabletAlias topo.TabletAlias) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } return updateReplicationGraphForPromotedSlave(ts, tablet) }
// Make this external, since these transitions need to be forced from time to time. func SetBlacklistedTables(ts topo.Server, tabletAlias topo.TabletAlias, tables []string) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } tablet.BlacklistedTables = tables return topo.UpdateTablet(ts, tablet) }
// Make this external, since these transitions need to be forced from time to time. func ChangeType(ts topo.Server, tabletAlias topo.TabletAlias, newType topo.TabletType, runHooks bool) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } if !topo.IsTrivialTypeChange(tablet.Type, newType) || !topo.IsValidTypeChange(tablet.Type, newType) { return fmt.Errorf("cannot change tablet type %v -> %v %v", tablet.Type, newType, tabletAlias) } if runHooks { // Only run the preflight_serving_type hook when // transitioning from non-serving to serving. if !topo.IsInServingGraph(tablet.Type) && topo.IsInServingGraph(newType) { if err := hook.NewSimpleHook("preflight_serving_type").ExecuteOptional(); err != nil { return err } } } tablet.Type = newType if newType == topo.TYPE_IDLE { if tablet.Parent.IsZero() { si, err := ts.GetShard(tablet.Keyspace, tablet.Shard) if err != nil { return err } rec := concurrency.AllErrorRecorder{} wg := sync.WaitGroup{} for _, cell := range si.Cells { wg.Add(1) go func(cell string) { defer wg.Done() sri, err := ts.GetShardReplication(cell, tablet.Keyspace, tablet.Shard) if err != nil { log.Warningf("Cannot check cell %v for extra replication paths, assuming it's good", cell) return } for _, rl := range sri.ReplicationLinks { if rl.Parent == tabletAlias { rec.RecordError(fmt.Errorf("Still have a ReplicationLink in cell %v", cell)) } } }(cell) } wg.Wait() if rec.HasErrors() { return rec.Error() } } tablet.Parent = topo.TabletAlias{} tablet.Keyspace = "" tablet.Shard = "" tablet.KeyRange = key.KeyRange{} } return topo.UpdateTablet(ts, tablet) }
func (th *tabletHealth) stream(ctx context.Context, ts topo.Server, tabletAlias *topodatapb.TabletAlias) (err error) { defer func() { th.mu.Lock() th.err = err th.mu.Unlock() close(th.done) }() ti, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return err } ep, err := topo.TabletEndPoint(ti.Tablet) if err != nil { return err } // Pass in a tablet type that is not UNKNOWN, so we don't ask // for sessionId. conn, err := tabletconn.GetDialer()(ctx, ep, "", "", topodatapb.TabletType_MASTER, 30*time.Second) if err != nil { return err } defer conn.Close() stream, err := conn.StreamHealth(ctx) if err != nil { return err } first := true for time.Since(th.lastAccessed()) < *tabletHealthKeepAlive { select { case <-ctx.Done(): return ctx.Err() default: } result, err := stream.Recv() if err != nil { return err } th.mu.Lock() th.result = result th.mu.Unlock() if first { // We got the first result, so we're ready to be accessed. close(th.ready) first = false } } return nil }
// handleExplorerRedirect returns the redirect target URL. func handleExplorerRedirect(ctx context.Context, ts topo.Server, r *http.Request) (string, error) { keyspace := r.FormValue("keyspace") shard := r.FormValue("shard") cell := r.FormValue("cell") switch r.FormValue("type") { case "keyspace": if keyspace == "" { return "", errors.New("keyspace is required for this redirect") } return appPrefix + "#/keyspaces/", nil case "shard": if keyspace == "" || shard == "" { return "", errors.New("keyspace and shard are required for this redirect") } return appPrefix + fmt.Sprintf("#/shard/%s/%s", keyspace, shard), nil case "srv_keyspace": if keyspace == "" || cell == "" { return "", errors.New("keyspace and cell are required for this redirect") } return appPrefix + "#/keyspaces/", nil case "srv_shard": if keyspace == "" || shard == "" || cell == "" { return "", errors.New("keyspace, shard, and cell are required for this redirect") } return appPrefix + fmt.Sprintf("#/shard/%s/%s", keyspace, shard), nil case "srv_type": tabletType := r.FormValue("tablet_type") if keyspace == "" || shard == "" || cell == "" || tabletType == "" { return "", errors.New("keyspace, shard, cell, and tablet_type are required for this redirect") } return appPrefix + fmt.Sprintf("#/shard/%s/%s", keyspace, shard), nil case "tablet": alias := r.FormValue("alias") if alias == "" { return "", errors.New("alias is required for this redirect") } tabletAlias, err := topoproto.ParseTabletAlias(alias) if err != nil { return "", fmt.Errorf("bad tablet alias %q: %v", alias, err) } ti, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return "", fmt.Errorf("can't get tablet %q: %v", alias, err) } return appPrefix + fmt.Sprintf("#/shard/%s/%s", ti.Keyspace, ti.Shard), nil case "replication": if keyspace == "" || shard == "" || cell == "" { return "", errors.New("keyspace, shard, and cell are required for this redirect") } return appPrefix + fmt.Sprintf("#/shard/%s/%s", keyspace, shard), nil default: return "", errors.New("bad redirect type") } }
// Scrap will update the tablet type to 'Scrap', and remove it from // the serving graph. // // 'force' means we are not on the tablet being scrapped, so it is // probably dead. So if 'force' is true, we will also remove pending // remote actions. And if 'force' is false, we also run an optional // hook. func Scrap(ts topo.Server, tabletAlias topo.TabletAlias, force bool) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } // If you are already scrap, skip updating replication data. It won't // be there anyway. wasAssigned := tablet.IsAssigned() tablet.Type = topo.TYPE_SCRAP tablet.Parent = topo.TabletAlias{} // Update the tablet first, since that is canonical. err = topo.UpdateTablet(ts, tablet) if err != nil { return err } // Remove any pending actions. Presumably forcing a scrap // means you don't want the agent doing anything and the // machine requires manual attention. if force { err := ts.PurgeTabletActions(tabletAlias, actionnode.ActionNodeCanBePurged) if err != nil { log.Warningf("purge actions failed: %v", err) } } if wasAssigned { err = topo.DeleteTabletReplicationData(ts, tablet.Tablet) if err != nil { if err == topo.ErrNoNode { log.V(6).Infof("no ShardReplication object for cell %v", tablet.Alias.Cell) err = nil } if err != nil { log.Warningf("remove replication data for %v failed: %v", tablet.Alias, err) } } } // run a hook for final cleanup, only in non-force mode. // (force mode executes on the vtctl side, not on the vttablet side) if !force { hk := hook.NewSimpleHook("postflight_scrap") ConfigureTabletHook(hk, tablet.Alias) if hookErr := hk.ExecuteOptional(); hookErr != nil { // we don't want to return an error, the server // is already in bad shape probably. log.Warningf("Scrap: postflight_scrap failed: %v", hookErr) } } return nil }
func (th *tabletHealth) stream(ctx context.Context, ts topo.Server, tabletAlias topo.TabletAlias) (err error) { defer func() { th.mu.Lock() th.err = err th.mu.Unlock() close(th.done) }() ti, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return err } ep, err := ti.EndPoint() if err != nil { return err } // pass in empty keyspace and shard to not ask for sessionId conn, err := tabletconn.GetDialer()(ctx, *ep, "", "", 30*time.Second) if err != nil { return err } defer conn.Close() stream, errFunc, err := conn.StreamHealth(ctx) if err != nil { return err } first := true for time.Since(th.lastAccessed()) < *tabletHealthKeepAlive { select { case <-ctx.Done(): return ctx.Err() case result, ok := <-stream: if !ok { return errFunc() } th.mu.Lock() th.result = result th.mu.Unlock() if first { // We got the first result, so we're ready to be accessed. close(th.ready) first = false } } } return nil }
// CopyTablets will create the tablets in the destination topo func CopyTablets(fromTS, toTS topo.Server) { cells, err := fromTS.GetKnownCells() if err != nil { log.Fatalf("fromTS.GetKnownCells: %v", err) } wg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} for _, cell := range cells { wg.Add(1) go func(cell string) { defer wg.Done() tabletAliases, err := fromTS.GetTabletsByCell(cell) if err != nil { rec.RecordError(fmt.Errorf("GetTabletsByCell(%v): %v", cell, err)) } else { for _, tabletAlias := range tabletAliases { wg.Add(1) go func(tabletAlias topo.TabletAlias) { defer wg.Done() // read the source tablet ti, err := fromTS.GetTablet(tabletAlias) if err != nil { rec.RecordError(fmt.Errorf("GetTablet(%v): %v", tabletAlias, err)) return } // try to create the destination err = toTS.CreateTablet(ti.Tablet) if err == topo.ErrNodeExists { // update the destination tablet log.Warningf("tablet %v already exists, updating it", tabletAlias) err = toTS.UpdateTabletFields(ti.Alias, func(t *topo.Tablet) error { *t = *ti.Tablet return nil }) } if err != nil { rec.RecordError(fmt.Errorf("CreateTablet(%v): %v", tabletAlias, err)) return } }(tabletAlias) } } }(cell) } wg.Wait() if rec.HasErrors() { log.Fatalf("copyTablets failed: %v", rec.Error()) } }
func (th *tabletHealth) stream(ctx context.Context, ts topo.Server, tabletAlias *topodatapb.TabletAlias) (err error) { defer func() { th.mu.Lock() th.err = err th.mu.Unlock() close(th.done) }() ti, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return err } conn, err := tabletconn.GetDialer()(ti.Tablet, 30*time.Second) if err != nil { return err } defer conn.Close(ctx) stream, err := conn.StreamHealth(ctx) if err != nil { return err } first := true for time.Since(th.lastAccessed()) < *tabletHealthKeepAlive { select { case <-ctx.Done(): return ctx.Err() default: } result, err := stream.Recv() if err != nil { return err } th.mu.Lock() th.result = result th.mu.Unlock() if first { // We got the first result, so we're ready to be accessed. close(th.ready) first = false } } return nil }
func SlaveWasPromoted(ts topo.Server, mysqlDaemon mysqlctl.MysqlDaemon, tabletAlias topo.TabletAlias) error { // We first check we don't have a master any more. // If we do, it probably means we're not *the* master, and something // is really wrong. masterAddr, err := mysqlDaemon.GetMasterAddr() if err != mysqlctl.ErrNotSlave { return fmt.Errorf("new master is a slave: %v %v", masterAddr, err) } tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } return updateReplicationGraphForPromotedSlave(ts, mysqlDaemon, tablet) }
func SlaveWasRestarted(ts topo.Server, mysqlDaemon mysqlctl.MysqlDaemon, tabletAlias topo.TabletAlias, swrd *actionnode.SlaveWasRestartedArgs) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } // check the reparent actually worked masterAddr, err := mysqlDaemon.GetMasterAddr() if err != nil { return err } if masterAddr != swrd.ExpectedMasterAddr && masterAddr != swrd.ExpectedMasterIpAddr { log.Errorf("SlaveWasRestarted found unexpected master %v for %v (was expecting %v or %v)", masterAddr, tabletAlias, swrd.ExpectedMasterAddr, swrd.ExpectedMasterIpAddr) // Disabled for now // if swrd.ContinueOnUnexpectedMaster { // log.Errorf("ContinueOnUnexpectedMaster is set, we keep going anyway") // } else if swrd.ScrapStragglers { return Scrap(ts, tablet.Alias, false) } else { return fmt.Errorf("Unexpected master %v for %v (was expecting %v or %v)", masterAddr, tabletAlias, swrd.ExpectedMasterAddr, swrd.ExpectedMasterIpAddr) } } // Once this action completes, update authoritive tablet node first. tablet.Parent = swrd.Parent if tablet.Type == topo.TYPE_MASTER { tablet.Type = topo.TYPE_SPARE tablet.State = topo.STATE_READ_ONLY } err = topo.UpdateTablet(ts, tablet) if err != nil { return err } // Update the new tablet location in the replication graph now that // we've updated the tablet. err = topo.CreateTabletReplicationData(ts, tablet.Tablet) if err != nil && err != topo.ErrNodeExists { return err } return nil }
// Scrap will update the tablet type to 'Scrap', and remove it from // the serving graph. // // 'force' means we are not on the tablet being scrapped, so it is // probably dead. So if 'force' is true, we will also remove pending // remote actions. And if 'force' is false, we also run an optional // hook. func Scrap(ctx context.Context, ts topo.Server, tabletAlias topo.TabletAlias, force bool) error { tablet, err := ts.GetTablet(ctx, tabletAlias) if err != nil { return err } // If you are already scrap, skip updating replication data. It won't // be there anyway. wasAssigned := tablet.IsAssigned() tablet.Type = topo.TYPE_SCRAP // Update the tablet first, since that is canonical. err = topo.UpdateTablet(ctx, ts, tablet) if err != nil { return err } if wasAssigned { err = topo.DeleteTabletReplicationData(ctx, ts, tablet.Tablet) if err != nil { if err == topo.ErrNoNode { log.V(6).Infof("no ShardReplication object for cell %v", tablet.Alias.Cell) err = nil } if err != nil { log.Warningf("remove replication data for %v failed: %v", tablet.Alias, err) } } } // run a hook for final cleanup, only in non-force mode. // (force mode executes on the vtctl side, not on the vttablet side) if !force { hk := hook.NewSimpleHook("postflight_scrap") ConfigureTabletHook(hk, tablet.Alias) if hookErr := hk.ExecuteOptional(); hookErr != nil { // we don't want to return an error, the server // is already in bad shape probably. log.Warningf("Scrap: postflight_scrap failed: %v", hookErr) } } return nil }
// Make this external, since these transitions need to be forced from time to time. func ChangeType(ts topo.Server, tabletAlias topo.TabletAlias, newType topo.TabletType, runHooks bool) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } if !topo.IsTrivialTypeChange(tablet.Type, newType) || !topo.IsValidTypeChange(tablet.Type, newType) { return fmt.Errorf("cannot change tablet type %v -> %v %v", tablet.Type, newType, tabletAlias) } if runHooks { // Only run the preflight_serving_type hook when // transitioning from non-serving to serving. if !topo.IsServingType(tablet.Type) && topo.IsServingType(newType) { if err := hook.NewSimpleHook("preflight_serving_type").ExecuteOptional(); err != nil { return err } } } tablet.Type = newType if newType == topo.TYPE_IDLE { if tablet.Parent.Uid == topo.NO_TABLET { // With a master the node cannot be set to idle unless we have already removed all of // the derived paths. The global replication path is a good indication that this has // been resolved. children, err := ts.GetReplicationPaths(tablet.Keyspace, tablet.Shard, tablet.ReplicationPath()) if err != nil && err != topo.ErrNoNode { return err } if err == nil && len(children) > 0 { return fmt.Errorf("cannot change tablet type %v -> %v - reparent action has not finished %v", tablet.Type, newType, tabletAlias) } } tablet.Parent = topo.TabletAlias{} tablet.Keyspace = "" tablet.Shard = "" tablet.KeyRange = key.KeyRange{} } return topo.UpdateTablet(ts, tablet) }
// NewQueryResultReaderForTablet creates a new QueryResultReader for // the provided tablet / sql query func NewQueryResultReaderForTablet(ts topo.Server, tabletAlias topo.TabletAlias, sql string) (*QueryResultReader, error) { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return nil, err } addr := fmt.Sprintf("%v:%v", tablet.IPAddr, tablet.Portmap["vt"]) rpcClient, err := bsonrpc.DialHTTP("tcp", addr, 30*time.Second, nil) if err != nil { return nil, err } var sessionInfo tproto.SessionInfo if err := rpcClient.Call("SqlQuery.GetSessionId", tproto.SessionParams{Keyspace: tablet.Keyspace, Shard: tablet.Shard}, &sessionInfo); err != nil { return nil, err } req := &tproto.Query{ Sql: sql, BindVariables: make(map[string]interface{}), TransactionId: 0, SessionId: sessionInfo.SessionId, } sr := make(chan *mproto.QueryResult, 1000) call := rpcClient.StreamGo("SqlQuery.StreamExecute", req, sr) // read the columns, or grab the error cols, ok := <-sr if !ok { return nil, fmt.Errorf("Cannot read Fields for query: %v", sql) } return &QueryResultReader{ Output: sr, Fields: cols.Fields, client: rpcClient, call: call, }, nil }
func addTablet(ctx context.Context, t *testing.T, ts topo.Server, uid int, cell string, tabletType topo.TabletType) *topo.TabletInfo { tablet := &topo.Tablet{ Alias: topo.TabletAlias{Cell: cell, Uid: uint32(uid)}, Hostname: fmt.Sprintf("%vbsr%v", cell, uid), IPAddr: fmt.Sprintf("212.244.218.%v", uid), Portmap: map[string]int{ "vt": 3333 + 10*uid, "mysql": 3334 + 10*uid, }, Keyspace: testKeyspace, Type: tabletType, Shard: testShard, } if err := topo.CreateTablet(ctx, ts, tablet); err != nil { t.Fatalf("CreateTablet: %v", err) } ti, err := ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Fatalf("GetTablet: %v", err) } return ti }
// Make this external, since in needs to be forced from time to time. func Scrap(ts topo.Server, tabletAlias topo.TabletAlias, force bool) error { tablet, err := ts.GetTablet(tabletAlias) if err != nil { return err } // If you are already scrap, skip deleting the path. It won't // be correct since the Parent will be cleared already. wasAssigned := tablet.IsAssigned() replicationPath := "" if wasAssigned { replicationPath = tablet.ReplicationPath() } tablet.Type = topo.TYPE_SCRAP tablet.Parent = topo.TabletAlias{} // Update the tablet first, since that is canonical. err = topo.UpdateTablet(ts, tablet) if err != nil { return err } // Remove any pending actions. Presumably forcing a scrap means you don't // want the agent doing anything and the machine requires manual attention. if force { err := ts.PurgeTabletActions(tabletAlias, ActionNodeCanBePurged) if err != nil { log.Warningf("purge actions failed: %v", err) } } if wasAssigned { err = ts.DeleteReplicationPath(tablet.Keyspace, tablet.Shard, replicationPath) if err != nil { switch err { case topo.ErrNoNode: log.V(6).Infof("no replication path: %v", replicationPath) err = nil case topo.ErrNotEmpty: // If you are forcing the scrapping of a master, you can't update the // replication graph yet, since other nodes are still under the impression // they are slaved to this tablet. // If the node was not empty, we can't do anything about it - the replication // graph needs to be fixed by reparenting. If the action was forced, assume // the user knows best and squelch the error. if tablet.Parent.Uid == topo.NO_TABLET && force { err = nil } } if err != nil { log.Warningf("remove replication path failed: %v %v", replicationPath, err) } } } // run a hook for final cleanup, only in non-force mode. // (force mode executes on the vtctl side, not on the vttablet side) if !force { hk := hook.NewSimpleHook("postflight_scrap") configureTabletHook(hk, tablet.Alias()) if hookErr := hk.ExecuteOptional(); hookErr != nil { // we don't want to return an error, the server // is already in bad shape probably. log.Warningf("Scrap: postflight_scrap failed: %v", hookErr) } } return nil }
func CheckTablet(t *testing.T, ts topo.Server) { cell := getLocalCell(t, ts) tablet := &topo.Tablet{ Cell: cell, Uid: 1, Addr: "localhost:3333", MysqlAddr: "localhost:3334", MysqlIpAddr: "10.11.12.13:3334", Alias: topo.TabletAlias{Cell: cell, Uid: 1}, Hostname: "localhost", IPAddr: "10.11.12.13", Portmap: map[string]int{ "vt": 3333, "mysql": 3334, }, Tags: map[string]string{"tag": "value"}, Keyspace: "test_keyspace", Type: topo.TYPE_MASTER, State: topo.STATE_READ_WRITE, KeyRange: newKeyRange("-10"), BlacklistedTables: []string{"black1", "black2"}, } if err := ts.CreateTablet(tablet); err != nil { t.Errorf("CreateTablet: %v", err) } if err := ts.CreateTablet(tablet); err != topo.ErrNodeExists { t.Errorf("CreateTablet(again): %v", err) } if _, err := ts.GetTablet(topo.TabletAlias{Cell: cell, Uid: 666}); err != topo.ErrNoNode { t.Errorf("GetTablet(666): %v", err) } ti, err := ts.GetTablet(tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } if eq, err := tabletEqual(ti.Tablet, tablet); err != nil { t.Errorf("cannot compare tablets: %v", err) } else if !eq { t.Errorf("put and got tablets are not identical:\n%#v\n%#v", tablet, ti.Tablet) } if _, err := ts.GetTabletsByCell("666"); err != topo.ErrNoNode { t.Errorf("GetTabletsByCell(666): %v", err) } inCell, err := ts.GetTabletsByCell(cell) if err != nil { t.Errorf("GetTabletsByCell: %v", err) } if len(inCell) != 1 || inCell[0] != tablet.Alias { t.Errorf("GetTabletsByCell: want [%v], got %v", tablet.Alias, inCell) } ti.State = topo.STATE_READ_ONLY if err := topo.UpdateTablet(ts, ti); err != nil { t.Errorf("UpdateTablet: %v", err) } ti, err = ts.GetTablet(tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } if want := topo.STATE_READ_ONLY; ti.State != want { t.Errorf("ti.State: want %v, got %v", want, ti.State) } if err := ts.UpdateTabletFields(tablet.Alias, func(t *topo.Tablet) error { t.State = topo.STATE_READ_WRITE return nil }); err != nil { t.Errorf("UpdateTabletFields: %v", err) } ti, err = ts.GetTablet(tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } if want := topo.STATE_READ_WRITE; ti.State != want { t.Errorf("ti.State: want %v, got %v", want, ti.State) } if err := ts.DeleteTablet(tablet.Alias); err != nil { t.Errorf("DeleteTablet: %v", err) } if err := ts.DeleteTablet(tablet.Alias); err != topo.ErrNoNode { t.Errorf("DeleteTablet(again): %v", err) } if _, err := ts.GetTablet(tablet.Alias); err != topo.ErrNoNode { t.Errorf("GetTablet: expected error, tablet was deleted: %v", err) } }
// initTabletMap creates the action agents and associated data structures // for all tablets, based on the vttest proto parameter. func initTabletMap(ts topo.Server, topoProto string, mysqld mysqlctl.MysqlDaemon, dbcfgs dbconfigs.DBConfigs, schemaDir string, mycnf *mysqlctl.Mycnf) error { // parse the input topology tpb := &vttestpb.VTTestTopology{} if err := proto.UnmarshalText(topoProto, tpb); err != nil { return fmt.Errorf("cannot parse topology: %v", err) } tabletMap = make(map[uint32]*tablet) ctx := context.Background() // disable publishing of stats from query service flag.Set("queryserver-config-enable-publish-stats", "false") // iterate through the keyspaces wr := wrangler.New(logutil.NewConsoleLogger(), ts, nil) var uid uint32 = 1 for _, kpb := range tpb.Keyspaces { keyspace := kpb.Name // First parse the ShardingColumnType. // Note if it's empty, we will return 'UNSET'. sct, err := key.ParseKeyspaceIDType(kpb.ShardingColumnType) if err != nil { return fmt.Errorf("parseKeyspaceIDType(%v) failed: %v", kpb.ShardingColumnType, err) } if kpb.ServedFrom != "" { // if we have a redirect, create a completely redirected // keyspace and no tablet if err := ts.CreateKeyspace(ctx, keyspace, &topodatapb.Keyspace{ ShardingColumnName: kpb.ShardingColumnName, ShardingColumnType: sct, ServedFroms: []*topodatapb.Keyspace_ServedFrom{ { TabletType: topodatapb.TabletType_MASTER, Keyspace: kpb.ServedFrom, }, { TabletType: topodatapb.TabletType_REPLICA, Keyspace: kpb.ServedFrom, }, { TabletType: topodatapb.TabletType_RDONLY, Keyspace: kpb.ServedFrom, }, }, }); err != nil { return fmt.Errorf("CreateKeyspace(%v) failed: %v", keyspace, err) } } else { // create a regular keyspace if err := ts.CreateKeyspace(ctx, keyspace, &topodatapb.Keyspace{ ShardingColumnName: kpb.ShardingColumnName, ShardingColumnType: sct, }); err != nil { return fmt.Errorf("CreateKeyspace(%v) failed: %v", keyspace, err) } // iterate through the shards for _, spb := range kpb.Shards { shard := spb.Name dbname := spb.DbNameOverride if dbname == "" { dbname = fmt.Sprintf("vt_%v_%v", keyspace, shard) } dbcfgs.App.DbName = dbname // create the master if err := createTablet(ctx, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_MASTER, mysqld, dbcfgs); err != nil { return err } uid++ // create a replica slave if err := createTablet(ctx, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_REPLICA, mysqld, dbcfgs); err != nil { return err } uid++ // create a rdonly slave if err := createTablet(ctx, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_RDONLY, mysqld, dbcfgs); err != nil { return err } uid++ } } // vschema for the keyspace if schemaDir != "" { f := path.Join(schemaDir, keyspace, "vschema.json") if _, err := os.Stat(f); err == nil { // load the vschema formal, err := vindexes.LoadFormalKeyspace(f) if err != nil { return fmt.Errorf("cannot load vschema file %v for keyspace %v: %v", f, keyspace, err) } if err := ts.SaveVSchema(ctx, keyspace, formal); err != nil { return fmt.Errorf("SaveVSchema(%v) failed: %v", keyspace, err) } } else { log.Infof("File %v doesn't exist, skipping vschema for keyspace %v", f, keyspace) } } // Rebuild the SrvKeyspace object, so we can support // range-based sharding queries, and export the redirects. if err := wr.RebuildKeyspaceGraph(ctx, keyspace, nil); err != nil { return fmt.Errorf("cannot rebuild %v: %v", keyspace, err) } } // Rebuild the SrvVSchema object if err := topotools.RebuildVSchema(ctx, wr.Logger(), ts, []string{cell}); err != nil { return fmt.Errorf("RebuildVSchemaGraph failed: %v", err) } // Register the tablet dialer for tablet server tabletconn.RegisterDialer("internal", dialer) *tabletconn.TabletProtocol = "internal" // Register the tablet manager client factory for tablet manager tmclient.RegisterTabletManagerClientFactory("internal", func() tmclient.TabletManagerClient { return &internalTabletManagerClient{} }) *tmclient.TabletManagerProtocol = "internal" // run healthcheck on all vttablets tmc := tmclient.NewTabletManagerClient() for _, tablet := range tabletMap { tabletInfo, err := ts.GetTablet(ctx, tablet.agent.TabletAlias) if err != nil { return fmt.Errorf("cannot find tablet: %+v", tablet.agent.TabletAlias) } tmc.RunHealthCheck(ctx, tabletInfo.Tablet) } return nil }
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { tabletHealthCache := newTabletHealthCache(ts) // Cells handleCollection("cells", func(r *http.Request) (interface{}, error) { if getItemPath(r.URL.Path) != "" { return nil, errors.New("cells can only be listed, not retrieved") } return ts.GetKnownCells(ctx) }) // Keyspaces handleCollection("keyspaces", func(r *http.Request) (interface{}, error) { keyspace := getItemPath(r.URL.Path) // List all keyspaces. if keyspace == "" { return ts.GetKeyspaces(ctx) } // Perform an action on a keyspace. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil } // Get the keyspace record. return ts.GetKeyspace(ctx, keyspace) }) // Shards handleCollection("shards", func(r *http.Request) (interface{}, error) { shardPath := getItemPath(r.URL.Path) if !strings.Contains(shardPath, "/") { return nil, fmt.Errorf("invalid shard path: %q", shardPath) } parts := strings.SplitN(shardPath, "/", 2) keyspace := parts[0] shard := parts[1] // List the shards in a keyspace. if shard == "" { return ts.GetShardNames(ctx, keyspace) } // Perform an action on a shard. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil } // Get the shard record. return ts.GetShard(ctx, keyspace, shard) }) // Tablets handleCollection("tablets", func(r *http.Request) (interface{}, error) { tabletPath := getItemPath(r.URL.Path) // List tablets based on query params. if tabletPath == "" { if err := r.ParseForm(); err != nil { return nil, err } shardRef := r.FormValue("shard") cell := r.FormValue("cell") if shardRef != "" { // Look up by keyspace/shard, and optionally cell. keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef) if err != nil { return nil, err } if cell != "" { return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell}) } return ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) } // Get all tablets in a cell. if cell == "" { return nil, errors.New("cell param required") } return ts.GetTabletsByCell(ctx, cell) } // Get tablet health. if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" { tabletAlias, err := topoproto.ParseTabletAlias(parts[0]) if err != nil { return nil, err } return tabletHealthCache.Get(ctx, tabletAlias) } tabletAlias, err := topoproto.ParseTabletAlias(tabletPath) if err != nil { return nil, err } // Perform an action on a tablet. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil } // Get the tablet record. return ts.GetTablet(ctx, tabletAlias) }) // Schema Change http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) { req := struct{ Keyspace, SQL string }{} if err := unmarshalRequest(r, &req); err != nil { httpErrorf(w, r, "can't unmarshal request: %v", err) return } executor := schemamanager.NewTabletExecutor( tmclient.NewTabletManagerClient(), ts) schemamanager.Run(ctx, schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor) }) }
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository, realtimeStats *realtimeStats) { tabletHealthCache := newTabletHealthCache(ts) tmClient := tmclient.NewTabletManagerClient() // Cells handleCollection("cells", func(r *http.Request) (interface{}, error) { if getItemPath(r.URL.Path) != "" { return nil, errors.New("cells can only be listed, not retrieved") } return ts.GetKnownCells(ctx) }) // Keyspaces handleCollection("keyspaces", func(r *http.Request) (interface{}, error) { keyspace := getItemPath(r.URL.Path) switch r.Method { case "GET": // List all keyspaces. if keyspace == "" { return ts.GetKeyspaces(ctx) } // Get the keyspace record. k, err := ts.GetKeyspace(ctx, keyspace) // Pass the embedded proto directly or jsonpb will panic. return k.Keyspace, err // Perform an action on a keyspace. case "POST": if keyspace == "" { return nil, errors.New("A POST request needs a keyspace in the URL") } if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("A POST request must specify action") } return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil default: return nil, fmt.Errorf("unsupported HTTP method: %v", r.Method) } }) // Shards handleCollection("shards", func(r *http.Request) (interface{}, error) { shardPath := getItemPath(r.URL.Path) if !strings.Contains(shardPath, "/") { return nil, fmt.Errorf("invalid shard path: %q", shardPath) } parts := strings.SplitN(shardPath, "/", 2) keyspace := parts[0] shard := parts[1] // List the shards in a keyspace. if shard == "" { return ts.GetShardNames(ctx, keyspace) } // Perform an action on a shard. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil } // Get the shard record. si, err := ts.GetShard(ctx, keyspace, shard) // Pass the embedded proto directly or jsonpb will panic. return si.Shard, err }) // SrvKeyspace handleCollection("srv_keyspace", func(r *http.Request) (interface{}, error) { keyspacePath := getItemPath(r.URL.Path) parts := strings.SplitN(keyspacePath, "/", 2) // Request was incorrectly formatted. if len(parts) != 2 { return nil, fmt.Errorf("invalid srvkeyspace path: %q expected path: /srv_keyspace/<cell>/<keyspace>", keyspacePath) } cell := parts[0] keyspace := parts[1] if cell == "local" { if *localCell == "" { return nil, fmt.Errorf("local cell requested, but not specified. Please set with -cell flag") } cell = *localCell } // If a keyspace is provided then return the specified srvkeyspace. if keyspace != "" { srvKeyspace, err := ts.GetSrvKeyspace(ctx, cell, keyspace) if err != nil { return nil, fmt.Errorf("Can't get server keyspace: %v", err) } return srvKeyspace, nil } // Else return the srvKeyspace from all keyspaces. srvKeyspaces := make(map[string]interface{}) keyspaceNamesList, err := ts.GetSrvKeyspaceNames(ctx, cell) if err != nil { return nil, fmt.Errorf("can't get list of SrvKeyspaceNames for cell %q: GetSrvKeyspaceNames returned: %v", cell, err) } for _, keyspaceName := range keyspaceNamesList { err := addSrvkeyspace(ctx, ts, cell, keyspaceName, srvKeyspaces) if err != nil { return nil, err } } return srvKeyspaces, nil }) // Tablets handleCollection("tablets", func(r *http.Request) (interface{}, error) { tabletPath := getItemPath(r.URL.Path) // List tablets based on query params. if tabletPath == "" { if err := r.ParseForm(); err != nil { return nil, err } shardRef := r.FormValue("shard") cell := r.FormValue("cell") if shardRef != "" { // Look up by keyspace/shard, and optionally cell. keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef) if err != nil { return nil, err } if cell != "" { result, err := ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell}) if err != nil && err != topo.ErrPartialResult { return result, err } return result, nil } result, err := ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) if err != nil && err != topo.ErrPartialResult { return result, err } return result, nil } // Get all tablets in a cell. if cell == "" { return nil, errors.New("cell param required") } return ts.GetTabletsByCell(ctx, cell) } // Get tablet health. if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" { tabletAlias, err := topoproto.ParseTabletAlias(parts[0]) if err != nil { return nil, err } return tabletHealthCache.Get(ctx, tabletAlias) } tabletAlias, err := topoproto.ParseTabletAlias(tabletPath) if err != nil { return nil, err } // Perform an action on a tablet. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil } // Get the tablet record. t, err := ts.GetTablet(ctx, tabletAlias) // Pass the embedded proto directly or jsonpb will panic. return t.Tablet, err }) // Healthcheck real time status per (cell, keyspace, tablet type, metric). handleCollection("tablet_statuses", func(r *http.Request) (interface{}, error) { targetPath := getItemPath(r.URL.Path) // Get the heatmap data based on query parameters. if targetPath == "" { if err := r.ParseForm(); err != nil { return nil, err } keyspace := r.FormValue("keyspace") cell := r.FormValue("cell") tabletType := r.FormValue("type") _, err := topoproto.ParseTabletType(tabletType) // Excluding the case where parse fails because all tabletTypes was chosen. if err != nil && tabletType != "all" { return nil, fmt.Errorf("invalid tablet type: %v ", err) } metric := r.FormValue("metric") // Setting default values if none was specified in the query params. if keyspace == "" { keyspace = "all" } if cell == "" { cell = "all" } if tabletType == "" { tabletType = "all" } if metric == "" { metric = "health" } if realtimeStats == nil { return nil, fmt.Errorf("realtimeStats not initialized") } heatmap, err := realtimeStats.heatmapData(keyspace, cell, tabletType, metric) if err != nil { return nil, fmt.Errorf("couldn't get heatmap data: %v", err) } return heatmap, nil } return nil, fmt.Errorf("invalid target path: %q expected path: ?keyspace=<keyspace>&cell=<cell>&type=<type>&metric=<metric>", targetPath) }) handleCollection("tablet_health", func(r *http.Request) (interface{}, error) { tabletPath := getItemPath(r.URL.Path) parts := strings.SplitN(tabletPath, "/", 2) // Request was incorrectly formatted. if len(parts) != 2 { return nil, fmt.Errorf("invalid tablet_health path: %q expected path: /tablet_health/<cell>/<uid>", tabletPath) } if realtimeStats == nil { return nil, fmt.Errorf("realtimeStats not initialized") } cell := parts[0] uidStr := parts[1] uid, err := topoproto.ParseUID(uidStr) if err != nil { return nil, fmt.Errorf("incorrect uid: %v", err) } tabletAlias := topodatapb.TabletAlias{ Cell: cell, Uid: uid, } tabletStat, err := realtimeStats.tabletStats(&tabletAlias) if err != nil { return nil, fmt.Errorf("could not get tabletStats: %v", err) } return tabletStat, nil }) handleCollection("topology_info", func(r *http.Request) (interface{}, error) { targetPath := getItemPath(r.URL.Path) // Retrieving topology information (keyspaces, cells, and types) based on query params. if targetPath == "" { if err := r.ParseForm(); err != nil { return nil, err } keyspace := r.FormValue("keyspace") cell := r.FormValue("cell") // Setting default values if none was specified in the query params. if keyspace == "" { keyspace = "all" } if cell == "" { cell = "all" } if realtimeStats == nil { return nil, fmt.Errorf("realtimeStats not initialized") } return realtimeStats.topologyInfo(keyspace, cell), nil } return nil, fmt.Errorf("invalid target path: %q expected path: ?keyspace=<keyspace>&cell=<cell>", targetPath) }) // Vtctl Command http.HandleFunc(apiPrefix+"vtctl/", func(w http.ResponseWriter, r *http.Request) { if err := acl.CheckAccessHTTP(r, acl.ADMIN); err != nil { httpErrorf(w, r, "Access denied") return } var args []string resp := struct { Error string Output string }{} if err := unmarshalRequest(r, &args); err != nil { httpErrorf(w, r, "can't unmarshal request: %v", err) return } logstream := logutil.NewMemoryLogger() wr := wrangler.New(logstream, ts, tmClient) // TODO(enisoc): Context for run command should be request-scoped. err := vtctl.RunCommand(ctx, wr, args) if err != nil { resp.Error = err.Error() } resp.Output = logstream.String() data, err := json.MarshalIndent(resp, "", " ") if err != nil { httpErrorf(w, r, "json error: %v", err) return } w.Header().Set("Content-Type", jsonContentType) w.Write(data) }) // Schema Change http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) { if err := acl.CheckAccessHTTP(r, acl.ADMIN); err != nil { httpErrorf(w, r, "Access denied") return } req := struct { Keyspace, SQL string SlaveTimeoutSeconds int }{} if err := unmarshalRequest(r, &req); err != nil { httpErrorf(w, r, "can't unmarshal request: %v", err) return } if req.SlaveTimeoutSeconds <= 0 { req.SlaveTimeoutSeconds = 10 } logger := logutil.NewCallbackLogger(func(ev *logutilpb.Event) { w.Write([]byte(logutil.EventString(ev))) }) wr := wrangler.New(logger, ts, tmClient) executor := schemamanager.NewTabletExecutor( wr, time.Duration(req.SlaveTimeoutSeconds)*time.Second) schemamanager.Run(ctx, schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor) }) }
// CheckTablet verifies the topo server API is correct for managing tablets. func CheckTablet(ctx context.Context, t *testing.T, ts topo.Server) { cell := getLocalCell(ctx, t, ts) tablet := &topo.Tablet{ Alias: topo.TabletAlias{Cell: cell, Uid: 1}, Hostname: "localhost", IPAddr: "10.11.12.13", Portmap: map[string]int{ "vt": 3333, "mysql": 3334, }, Tags: map[string]string{"tag": "value"}, Keyspace: "test_keyspace", Type: topo.TYPE_MASTER, KeyRange: newKeyRange("-10"), } if err := ts.CreateTablet(ctx, tablet); err != nil { t.Errorf("CreateTablet: %v", err) } if err := ts.CreateTablet(ctx, tablet); err != topo.ErrNodeExists { t.Errorf("CreateTablet(again): %v", err) } if _, err := ts.GetTablet(ctx, topo.TabletAlias{Cell: cell, Uid: 666}); err != topo.ErrNoNode { t.Errorf("GetTablet(666): %v", err) } ti, err := ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } if eq, err := tabletEqual(ti.Tablet, tablet); err != nil { t.Errorf("cannot compare tablets: %v", err) } else if !eq { t.Errorf("put and got tablets are not identical:\n%#v\n%#v", tablet, ti.Tablet) } if _, err := ts.GetTabletsByCell(ctx, "666"); err != topo.ErrNoNode { t.Errorf("GetTabletsByCell(666): %v", err) } inCell, err := ts.GetTabletsByCell(ctx, cell) if err != nil { t.Errorf("GetTabletsByCell: %v", err) } if len(inCell) != 1 || inCell[0] != tablet.Alias { t.Errorf("GetTabletsByCell: want [%v], got %v", tablet.Alias, inCell) } ti.Hostname = "remotehost" if err := topo.UpdateTablet(ctx, ts, ti); err != nil { t.Errorf("UpdateTablet: %v", err) } ti, err = ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } if want := "remotehost"; ti.Hostname != want { t.Errorf("ti.Hostname: want %v, got %v", want, ti.Hostname) } if err := topo.UpdateTabletFields(ctx, ts, tablet.Alias, func(t *topo.Tablet) error { t.Hostname = "anotherhost" return nil }); err != nil { t.Errorf("UpdateTabletFields: %v", err) } ti, err = ts.GetTablet(ctx, tablet.Alias) if err != nil { t.Errorf("GetTablet %v: %v", tablet.Alias, err) } if want := "anotherhost"; ti.Hostname != want { t.Errorf("ti.Hostname: want %v, got %v", want, ti.Hostname) } if err := ts.DeleteTablet(ctx, tablet.Alias); err != nil { t.Errorf("DeleteTablet: %v", err) } if err := ts.DeleteTablet(ctx, tablet.Alias); err != topo.ErrNoNode { t.Errorf("DeleteTablet(again): %v", err) } if _, err := ts.GetTablet(ctx, tablet.Alias); err != topo.ErrNoNode { t.Errorf("GetTablet: expected error, tablet was deleted: %v", err) } }
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { tabletHealthCache := newTabletHealthCache(ts) // Cells handleCollection("cells", func(r *http.Request) (interface{}, error) { if getItemPath(r.URL.Path) != "" { return nil, errors.New("cells can only be listed, not retrieved") } return ts.GetKnownCells(ctx) }) // Keyspaces handleCollection("keyspaces", func(r *http.Request) (interface{}, error) { keyspace := getItemPath(r.URL.Path) // List all keyspaces. if keyspace == "" { return ts.GetKeyspaces(ctx) } // Perform an action on a keyspace. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil } // Get the keyspace record. return ts.GetKeyspace(ctx, keyspace) }) // Shards handleCollection("shards", func(r *http.Request) (interface{}, error) { shardPath := getItemPath(r.URL.Path) if !strings.Contains(shardPath, "/") { return nil, fmt.Errorf("invalid shard path: %q", shardPath) } parts := strings.SplitN(shardPath, "/", 2) keyspace := parts[0] shard := parts[1] // List the shards in a keyspace. if shard == "" { return ts.GetShardNames(ctx, keyspace) } // Perform an action on a shard. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil } // Get the shard record. return ts.GetShard(ctx, keyspace, shard) }) // Tablets handleCollection("tablets", func(r *http.Request) (interface{}, error) { tabletPath := getItemPath(r.URL.Path) // List tablets based on query params. if tabletPath == "" { if err := r.ParseForm(); err != nil { return nil, err } shardRef := r.FormValue("shard") cell := r.FormValue("cell") if shardRef != "" { // Look up by keyspace/shard, and optionally cell. keyspace, shard, err := topoproto.ParseKeyspaceShard(shardRef) if err != nil { return nil, err } if cell != "" { return ts.FindAllTabletAliasesInShardByCell(ctx, keyspace, shard, []string{cell}) } return ts.FindAllTabletAliasesInShard(ctx, keyspace, shard) } // Get all tablets in a cell. if cell == "" { return nil, errors.New("cell param required") } return ts.GetTabletsByCell(ctx, cell) } // Get tablet health. if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" { tabletAlias, err := topoproto.ParseTabletAlias(parts[0]) if err != nil { return nil, err } return tabletHealthCache.Get(ctx, tabletAlias) } tabletAlias, err := topoproto.ParseTabletAlias(tabletPath) if err != nil { return nil, err } // Perform an action on a tablet. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil } // Get the tablet record. return ts.GetTablet(ctx, tabletAlias) }) // EndPoints handleCollection("endpoints", func(r *http.Request) (interface{}, error) { // We expect cell/keyspace/shard/tabletType. epPath := getItemPath(r.URL.Path) parts := strings.Split(epPath, "/") if len(parts) != 4 { return nil, fmt.Errorf("invalid cell/keyspace/shard/tabletType: %q", epPath) } if parts[3] == "" { // tabletType is empty, so list the tablet types. return ts.GetSrvTabletTypesPerShard(ctx, parts[0], parts[1], parts[2]) } tabletType, err := topoproto.ParseTabletType(parts[3]) if err != nil { return nil, fmt.Errorf("invalid tablet type %v: %v", parts[3], err) } // Get the endpoints object for a specific type. ep, _, err := ts.GetEndPoints(ctx, parts[0], parts[1], parts[2], tabletType) return ep, err }) // Schema Change http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) { req := struct{ Keyspace, SQL string }{} if err := unmarshalRequest(r, &req); err != nil { httpErrorf(w, r, "can't unmarshal request: %v", err) return } executor := schemamanager.NewTabletExecutor( tmclient.NewTabletManagerClient(), ts) schemamanager.Run(ctx, schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor) }) // VSchema http.HandleFunc(apiPrefix+"vschema/", func(w http.ResponseWriter, r *http.Request) { // Save VSchema if r.Method == "POST" { vschema, err := ioutil.ReadAll(r.Body) if err != nil { httpErrorf(w, r, "can't read request body: %v", err) return } if err := ts.SaveVSchema(ctx, string(vschema)); err != nil { httpErrorf(w, r, "can't save vschema: %v", err) } return } // Get VSchema vschema, err := ts.GetVSchema(ctx) if err != nil { httpErrorf(w, r, "can't get vschema: %v", err) return } w.Header().Set("Content-Type", jsonContentType) w.Write([]byte(vschema)) }) }