func TestQueryShard(t *testing.T) {
reflected, err := bson.Marshal(&reflectQueryShard{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
Shards: []string{"shard1", "shard2"},
TabletType: topo.TabletType("replica"),
Session: &commonSession,
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := QueryShard{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
Shards: []string{"shard1", "shard2"},
TabletType: topo.TabletType("replica"),
Session: &commonSession,
}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%#v, got\n%#v", want, got)
}
var unmarshalled QueryShard
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%#v, got \n%#v", custom, unmarshalled)
}
unexpected, err := bson.Marshal(&badQueryShard{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(unexpected, &unmarshalled)
want = "Unrecognized tag Extra"
if err == nil || want != err.Error() {
t.Errorf("want %v, got %v", want, err)
}
}
func TestSession(t *testing.T) {
reflected, err := bson.Marshal(&reflectSession{
InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := commonSession
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%#v, got\n%#v", want, got)
}
var unmarshalled Session
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%#v, got \n%#v", custom, unmarshalled)
}
unexpected, err := bson.Marshal(&badSession{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(unexpected, &unmarshalled)
want = "Unrecognized tag Extra"
if err == nil || want != err.Error() {
t.Errorf("want %v, got %v", want, err)
}
}
func getSrvKeyspace(rpcClient *rpcplus.Client, cell, keyspace string, verbose bool) {
req := &topo.GetSrvKeyspaceArgs{
Cell: cell,
Keyspace: keyspace,
}
reply := &topo.SrvKeyspace{}
if err := rpcClient.Call("TopoReader.GetSrvKeyspace", req, reply); err != nil {
log.Fatalf("TopoReader.GetSrvKeyspace error: %v", err)
}
if verbose {
tabletTypes := make([]string, 0, len(reply.Partitions))
for t, _ := range reply.Partitions {
tabletTypes = append(tabletTypes, string(t))
}
sort.Strings(tabletTypes)
for _, t := range tabletTypes {
println(fmt.Sprintf("Partitions[%v] =", t))
for i, s := range reply.Partitions[topo.TabletType(t)].Shards {
println(fmt.Sprintf(" Shards[%v]=%v", i, s.KeyRange.String()))
}
}
for i, s := range reply.Shards {
println(fmt.Sprintf("Shards[%v]=%v", i, s.KeyRange.String()))
}
for i, t := range reply.TabletTypes {
println(fmt.Sprintf("TabletTypes[%v] = %v", i, t))
}
}
}
// UnmarshalBson unmarshals QueryShard from buf.
func (qrs *QueryShard) UnmarshalBson(buf *bytes.Buffer) {
bson.Next(buf, 4)
kind := bson.NextByte(buf)
for kind != bson.EOO {
key := bson.ReadCString(buf)
switch key {
case "Sql":
qrs.Sql = bson.DecodeString(buf, kind)
case "BindVariables":
qrs.BindVariables = tproto.DecodeBindVariablesBson(buf, kind)
case "Keyspace":
qrs.Keyspace = bson.DecodeString(buf, kind)
case "TabletType":
qrs.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "Shards":
qrs.Shards = bson.DecodeStringArray(buf, kind)
case "Session":
qrs.Session = new(Session)
qrs.Session.UnmarshalBson(buf)
default:
panic(bson.NewBsonError("Unrecognized tag %s", key))
}
kind = bson.NextByte(buf)
}
}
func (sqs *StreamQueryKeyRange) UnmarshalBson(buf *bytes.Buffer) {
bson.Next(buf, 4)
kind := bson.NextByte(buf)
for kind != bson.EOO {
keyName := bson.ReadCString(buf)
switch keyName {
case "Sql":
sqs.Sql = bson.DecodeString(buf, kind)
case "BindVariables":
sqs.BindVariables = tproto.DecodeBindVariablesBson(buf, kind)
case "Keyspace":
sqs.Keyspace = bson.DecodeString(buf, kind)
case "KeyRange":
sqs.KeyRange = bson.DecodeString(buf, kind)
case "TabletType":
sqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "Session":
sqs.Session = new(Session)
sqs.Session.UnmarshalBson(buf)
default:
panic(bson.NewBsonError("Unrecognized tag %s", keyName))
}
kind = bson.NextByte(buf)
}
}
// for direct zk connection: vtzk://host:port/cell/keyspace/tabletType
// we always use a MetaConn, host and port are ignored.
// the driver name dictates if we use zk or zkocc, and streaming or not
//
// if user and password are specified in the URL, they will be used
// for each DB connection to the tablet's vttablet processes
func (driver *sDriver) Open(name string) (sc db.Conn, err error) {
if !strings.HasPrefix(name, "vtzk://") {
// add a default protocol talking to zk
name = "vtzk://" + name
}
u, err := url.Parse(name)
if err != nil {
return nil, err
}
dbi, tabletType := path.Split(u.Path)
dbi = strings.Trim(dbi, "/")
tabletType = strings.Trim(tabletType, "/")
cell, keyspace := path.Split(dbi)
cell = strings.Trim(cell, "/")
keyspace = strings.Trim(keyspace, "/")
var user, password string
if u.User != nil {
user = u.User.Username()
var ok bool
password, ok = u.User.Password()
if !ok {
return nil, fmt.Errorf("vt: need a password if a user is specified")
}
}
return Dial(driver.ts, cell, keyspace, topo.TabletType(tabletType), driver.stream, tablet.DefaultTimeout, user, password)
}
// UnmarshalBson unmarshals BatchQueryShard from buf.
func (bqs *BatchQueryShard) UnmarshalBson(buf *bytes.Buffer, kind byte) {
bson.VerifyObject(kind)
bson.Next(buf, 4)
kind = bson.NextByte(buf)
for kind != bson.EOO {
keyName := bson.ReadCString(buf)
switch keyName {
case "Queries":
bqs.Queries = tproto.DecodeQueriesBson(buf, kind)
case "Keyspace":
bqs.Keyspace = bson.DecodeString(buf, kind)
case "Shards":
bqs.Shards = bson.DecodeStringArray(buf, kind)
case "TabletType":
bqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "Session":
if kind != bson.Null {
bqs.Session = new(Session)
bqs.Session.UnmarshalBson(buf, kind)
}
default:
bson.Skip(buf, kind)
}
kind = bson.NextByte(buf)
}
}
func (sqs *StreamQueryKeyRange) UnmarshalBson(buf *bytes.Buffer, kind byte) {
bson.VerifyObject(kind)
bson.Next(buf, 4)
kind = bson.NextByte(buf)
for kind != bson.EOO {
keyName := bson.ReadCString(buf)
switch keyName {
case "Sql":
sqs.Sql = bson.DecodeString(buf, kind)
case "BindVariables":
sqs.BindVariables = tproto.DecodeBindVariablesBson(buf, kind)
case "Keyspace":
sqs.Keyspace = bson.DecodeString(buf, kind)
case "KeyRange":
sqs.KeyRange = bson.DecodeString(buf, kind)
case "TabletType":
sqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "Session":
if kind != bson.Null {
sqs.Session = new(Session)
sqs.Session.UnmarshalBson(buf, kind)
}
default:
bson.Skip(buf, kind)
}
kind = bson.NextByte(buf)
}
}
func TestHashResolve(t *testing.T) {
hind := NewHashIndex(TEST_SHARDED, new(sandboxTopo), "")
nn, _ := sqltypes.BuildNumeric("11")
ks, shards, err := hind.Resolve(topo.TabletType("master"), []interface{}{1, int32(2), int64(3), uint(4), uint32(5), uint64(6), nn})
if err != nil {
t.Error(err)
}
want := []string{"-20", "-20", "40-60", "c0-e0", "60-80", "e0-", "a0-c0"}
if !reflect.DeepEqual(shards, want) {
t.Errorf("got\n%#v, want\n%#v", shards, want)
}
if ks != TEST_SHARDED {
t.Errorf("got %v, want TEST_SHARDED", ks)
}
_, _, err = hind.Resolve(topo.TabletType("master"), []interface{}{"aa"})
wantErr := "unexpected type for aa: string"
if err == nil || err.Error() != wantErr {
t.Errorf("got %v, want %v", err, wantErr)
}
}
func (agent *ActionAgent) initHeathCheck() {
if !agent.IsRunningHealthCheck() {
log.Infof("No target_tablet_type specified, disabling any health check")
return
}
log.Infof("Starting periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType)
t := timer.NewTimer(*healthCheckInterval)
servenv.OnTerm(func() {
// When we enter lameduck mode, we want to not call
// the health check any more. After this returns, we
// are guaranteed to not call it.
log.Info("Stopping periodic health check timer")
t.Stop()
// Now we can finish up and force ourselves to not healthy.
agent.terminateHealthChecks(topo.TabletType(*targetTabletType))
})
t.Start(func() {
agent.runHealthCheck(topo.TabletType(*targetTabletType))
})
}
// handleExplorerRedirect returns the redirect target URL.
func handleExplorerRedirect(r *http.Request) (string, error) {
keyspace := r.FormValue("keyspace")
shard := r.FormValue("shard")
cell := r.FormValue("cell")
switch r.FormValue("type") {
case "keyspace":
if keyspace == "" {
return "", errors.New("keyspace is required for this redirect")
}
return explorer.GetKeyspacePath(keyspace), nil
case "shard":
if keyspace == "" || shard == "" {
return "", errors.New("keyspace and shard are required for this redirect")
}
return explorer.GetShardPath(keyspace, shard), nil
case "srv_keyspace":
if keyspace == "" || cell == "" {
return "", errors.New("keyspace and cell are required for this redirect")
}
return explorer.GetSrvKeyspacePath(cell, keyspace), nil
case "srv_shard":
if keyspace == "" || shard == "" || cell == "" {
return "", errors.New("keyspace, shard, and cell are required for this redirect")
}
return explorer.GetSrvShardPath(cell, keyspace, shard), nil
case "srv_type":
tabletType := r.FormValue("tablet_type")
if keyspace == "" || shard == "" || cell == "" || tabletType == "" {
return "", errors.New("keyspace, shard, cell, and tablet_type are required for this redirect")
}
return explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType)), nil
case "tablet":
alias := r.FormValue("alias")
if alias == "" {
return "", errors.New("alias is required for this redirect")
}
tabletAlias, err := topo.ParseTabletAliasString(alias)
if err != nil {
return "", fmt.Errorf("bad tablet alias %q: %v", alias, err)
}
return explorer.GetTabletPath(tabletAlias), nil
case "replication":
if keyspace == "" || shard == "" || cell == "" {
return "", errors.New("keyspace, shard, and cell are required for this redirect")
}
return explorer.GetReplicationSlaves(cell, keyspace, shard), nil
default:
return "", errors.New("bad redirect type")
}
}
func initHeathCheck(agent *tabletmanager.ActionAgent) {
if *targetTabletType == "" {
log.Infof("No target_tablet_type specified, disabling any health check")
return
}
log.Infof("Starting up periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType)
go func() {
t := time.NewTicker(*healthCheckInterval)
for _ = range t.C {
agent.RunHealthCheck(topo.TabletType(*targetTabletType))
}
}()
}
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) {
zkSgShardPath := zkPathForVtShard(cell, keyspace, shard)
children, _, err := zkts.zconn.Children(zkSgShardPath)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
err = topo.ErrNoNode
}
return nil, err
}
result := make([]topo.TabletType, len(children))
for i, tt := range children {
result[i] = topo.TabletType(tt)
}
return result, nil
}
func (spm *SessionParams) UnmarshalBson(buf *bytes.Buffer) {
bson.Next(buf, 4)
kind := bson.NextByte(buf)
for kind != bson.EOO {
key := bson.ReadCString(buf)
switch key {
case "TabletType":
spm.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
default:
panic(bson.NewBsonError("Unrecognized tag %s", key))
}
kind = bson.NextByte(buf)
}
}
func TestSessionParams(t *testing.T) {
reflected, err := bson.Marshal(&reflectSessionParams{topo.TabletType("master")})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := SessionParams{topo.TabletType("master")}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%#v, got\n%#v", want, got)
}
var unmarshalled SessionParams
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if custom != unmarshalled {
t.Errorf("want %v, got %#v", custom, unmarshalled)
}
unexpected, err := bson.Marshal(&badSessionParams{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(unexpected, &unmarshalled)
want = "Unrecognized tag Extra"
if err == nil || want != err.Error() {
t.Errorf("want %v, got %v", want, err)
}
}
func getEndPoints(rpcClient *rpcplus.Client, cell, keyspace, shard, tabletType string, verbose bool) {
req := &topo.GetEndPointsArgs{
Cell: cell,
Keyspace: keyspace,
Shard: shard,
TabletType: topo.TabletType(tabletType),
}
reply := &topo.EndPoints{}
if err := rpcClient.Call("TopoReader.GetEndPoints", req, reply); err != nil {
log.Fatalf("TopoReader.GetEndPoints error: %v", err)
}
if verbose {
for i, e := range reply.Entries {
println(fmt.Sprintf("Entries[%v] = %v %v", i, e.Uid, e.Host))
}
}
}
// for direct zk connection: vtzk://host:port/cell/keyspace/tabletType
// we always use a MetaConn, host and port are ignored.
// the driver name dictates if we use zk or zkocc, and streaming or not
func (driver *sDriver) Open(name string) (sc db.Conn, err error) {
if !strings.HasPrefix(name, "vtzk://") {
// add a default protocol talking to zk
name = "vtzk://" + name
}
u, err := url.Parse(name)
if err != nil {
return nil, err
}
dbi, tabletType := path.Split(u.Path)
dbi = strings.Trim(dbi, "/")
tabletType = strings.Trim(tabletType, "/")
cell, keyspace := path.Split(dbi)
cell = strings.Trim(cell, "/")
keyspace = strings.Trim(keyspace, "/")
return Dial(driver.ts, cell, keyspace, topo.TabletType(tabletType), driver.stream, tablet.DefaultTimeout)
}
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) {
zkSgShardPath := zkPathForVtShard(cell, keyspace, shard)
children, _, err := zkts.zconn.Children(zkSgShardPath)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
err = topo.ErrNoNode
}
return nil, err
}
result := make([]topo.TabletType, 0, len(children))
for _, tt := range children {
// these two are used for locking
if tt == "action" || tt == "actionlog" {
continue
}
result = append(result, topo.TabletType(tt))
}
return result, nil
}
// GetSrvTabletTypesPerShard implements topo.Server.
func (s *Server) GetSrvTabletTypesPerShard(ctx context.Context, cellName, keyspace, shard string) ([]topo.TabletType, error) {
cell, err := s.getCell(cellName)
if err != nil {
return nil, err
}
resp, err := cell.Get(srvShardDirPath(keyspace, shard), false /* sort */, false /* recursive */)
if err != nil {
return nil, convertError(err)
}
if resp.Node == nil {
return nil, ErrBadResponse
}
tabletTypes := make([]topo.TabletType, 0, len(resp.Node.Nodes))
for _, n := range resp.Node.Nodes {
tabletTypes = append(tabletTypes, topo.TabletType(path.Base(n.Key)))
}
return tabletTypes, nil
}
// UnmarshalBson unmarshals ShardSession from buf.
func (shardSession *ShardSession) UnmarshalBson(buf *bytes.Buffer) {
bson.Next(buf, 4)
kind := bson.NextByte(buf)
for kind != bson.EOO {
key := bson.ReadCString(buf)
switch key {
case "Keyspace":
shardSession.Keyspace = bson.DecodeString(buf, kind)
case "Shard":
shardSession.Shard = bson.DecodeString(buf, kind)
case "TabletType":
shardSession.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "TransactionId":
shardSession.TransactionId = bson.DecodeInt64(buf, kind)
default:
panic(bson.NewBsonError("Unrecognized tag %s", key))
}
kind = bson.NextByte(buf)
}
}
// UnmarshalBson unmarshals ShardSession from buf.
func (shardSession *ShardSession) UnmarshalBson(buf *bytes.Buffer, kind byte) {
bson.VerifyObject(kind)
bson.Next(buf, 4)
kind = bson.NextByte(buf)
for kind != bson.EOO {
keyName := bson.ReadCString(buf)
switch keyName {
case "Keyspace":
shardSession.Keyspace = bson.DecodeString(buf, kind)
case "Shard":
shardSession.Shard = bson.DecodeString(buf, kind)
case "TabletType":
shardSession.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "TransactionId":
shardSession.TransactionId = bson.DecodeInt64(buf, kind)
default:
bson.Skip(buf, kind)
}
kind = bson.NextByte(buf)
}
}
// UnmarshalBson unmarshals BatchQueryShard from buf.
func (bqs *BatchQueryShard) UnmarshalBson(buf *bytes.Buffer) {
bson.Next(buf, 4)
kind := bson.NextByte(buf)
for kind != bson.EOO {
keyName := bson.ReadCString(buf)
switch keyName {
case "Queries":
bqs.Queries = tproto.DecodeQueriesBson(buf, kind)
case "Keyspace":
bqs.Keyspace = bson.DecodeString(buf, kind)
case "Shards":
bqs.Shards = bson.DecodeStringArray(buf, kind)
case "TabletType":
bqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind))
case "Session":
bqs.Session = new(Session)
bqs.Session.UnmarshalBson(buf)
default:
panic(bson.NewBsonError("Unrecognized tag %s", keyName))
}
kind = bson.NextByte(buf)
}
}
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) {
tabletHealthCache := newTabletHealthCache(ts)
// Cells
handleCollection("cells", func(r *http.Request) (interface{}, error) {
if getItemPath(r.URL.Path) != "" {
return nil, errors.New("cells can only be listed, not retrieved")
}
return ts.GetKnownCells(ctx)
})
// Keyspaces
handleCollection("keyspaces", func(r *http.Request) (interface{}, error) {
keyspace := getItemPath(r.URL.Path)
// List all keyspaces.
if keyspace == "" {
return ts.GetKeyspaces(ctx)
}
// Perform an action on a keyspace.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil
}
// Get the keyspace record.
return ts.GetKeyspace(ctx, keyspace)
})
// Shards
handleCollection("shards", func(r *http.Request) (interface{}, error) {
shardPath := getItemPath(r.URL.Path)
if !strings.Contains(shardPath, "/") {
return nil, fmt.Errorf("invalid shard path: %q", shardPath)
}
parts := strings.SplitN(shardPath, "/", 2)
keyspace := parts[0]
shard := parts[1]
// List the shards in a keyspace.
if shard == "" {
return ts.GetShardNames(ctx, keyspace)
}
// Perform an action on a shard.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil
}
// Get the shard record.
return ts.GetShard(ctx, keyspace, shard)
})
// Tablets
handleCollection("tablets", func(r *http.Request) (interface{}, error) {
tabletPath := getItemPath(r.URL.Path)
// List tablets based on query params.
if tabletPath == "" {
if err := r.ParseForm(); err != nil {
return nil, err
}
shardRef := r.FormValue("shard")
cell := r.FormValue("cell")
if shardRef != "" {
// Look up by keyspace/shard, and optionally cell.
keyspace, shard, err := topo.ParseKeyspaceShardString(shardRef)
if err != nil {
return nil, err
}
if cell != "" {
return topo.FindAllTabletAliasesInShardByCell(ctx, ts, keyspace, shard, []string{cell})
}
return topo.FindAllTabletAliasesInShard(ctx, ts, keyspace, shard)
}
// Get all tablets in a cell.
if cell == "" {
return nil, errors.New("cell param required")
}
return ts.GetTabletsByCell(ctx, cell)
}
// Get tablet health.
if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" {
tabletAlias, err := topo.ParseTabletAliasString(parts[0])
if err != nil {
return nil, err
}
return tabletHealthCache.Get(ctx, tabletAlias)
}
tabletAlias, err := topo.ParseTabletAliasString(tabletPath)
if err != nil {
return nil, err
}
// Perform an action on a tablet.
if r.Method == "POST" {
if err := r.ParseForm(); err != nil {
return nil, err
}
action := r.FormValue("action")
if action == "" {
return nil, errors.New("must specify action")
}
return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil
}
// Get the tablet record.
return ts.GetTablet(ctx, tabletAlias)
})
// EndPoints
handleCollection("endpoints", func(r *http.Request) (interface{}, error) {
// We expect cell/keyspace/shard/tabletType.
epPath := getItemPath(r.URL.Path)
parts := strings.Split(epPath, "/")
if len(parts) != 4 {
return nil, fmt.Errorf("invalid cell/keyspace/shard/tabletType: %q", epPath)
}
if parts[3] == "" {
// tabletType is empty, so list the tablet types.
return ts.GetSrvTabletTypesPerShard(ctx, parts[0], parts[1], parts[2])
}
// Get the endpoints object for a specific type.
ep, _, err := ts.GetEndPoints(ctx, parts[0], parts[1], parts[2], topo.TabletType(parts[3]))
return ep, err
})
// Schema Change
http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) {
req := struct{ Keyspace, SQL string }{}
if err := unmarshalRequest(r, &req); err != nil {
httpErrorf(w, r, "can't unmarshal request: %v", err)
return
}
executor := schemamanager.NewTabletExecutor(
tmclient.NewTabletManagerClient(),
ts)
schemamanager.Run(ctx,
schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor)
})
// VSchema
http.HandleFunc(apiPrefix+"vschema/", func(w http.ResponseWriter, r *http.Request) {
schemafier, ok := ts.(topo.Schemafier)
if !ok {
httpErrorf(w, r, "%T doesn't support schemafier API", ts)
return
}
// Save VSchema
if r.Method == "POST" {
vschema, err := ioutil.ReadAll(r.Body)
if err != nil {
httpErrorf(w, r, "can't read request body: %v", err)
return
}
if err := schemafier.SaveVSchema(ctx, string(vschema)); err != nil {
httpErrorf(w, r, "can't save vschema: %v", err)
}
return
}
// Get VSchema
vschema, err := schemafier.GetVSchema(ctx)
if err != nil {
httpErrorf(w, r, "can't get vschema: %v", err)
return
}
w.Header().Set("Content-Type", jsonContentType)
w.Write([]byte(vschema))
})
}
func TestKeyspaceIdBatchQuery(t *testing.T) {
reflected, err := bson.Marshal(&reflectKeyspaceIdBatchQuery{
Queries: []reflectBoundKeyspaceIdQuery{{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")},
}},
Session: &Session{InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
},
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := KeyspaceIdBatchQuery{
Queries: []BoundKeyspaceIdQuery{{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")},
}},
Session: &commonSession,
}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
}
var unmarshalled KeyspaceIdBatchQuery
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled)
}
extra, err := bson.Marshal(&extraKeyspaceIdBatchQuery{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(extra, &unmarshalled)
if err != nil {
t.Error(err)
}
}
"reflect"
"testing"
"github.com/youtube/vitess/go/bson"
mproto "github.com/youtube/vitess/go/mysql/proto"
"github.com/youtube/vitess/go/sqltypes"
kproto "github.com/youtube/vitess/go/vt/key"
"github.com/youtube/vitess/go/vt/topo"
)
var commonSession = Session{
InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
}
type reflectSession struct {
InTransaction bool
ShardSessions []*ShardSession
}
type extraSession struct {
func (wr *Wrangler) exportVtnsToZkns(ctx context.Context, zconn zk.Conn, vtnsAddrPath, zknsAddrPath string) ([]string, error) {
zknsPaths := make([]string, 0, 32)
parts := strings.Split(vtnsAddrPath, "/")
if len(parts) != 8 && len(parts) != 9 {
return nil, fmt.Errorf("Invalid leaf zk path: %v", vtnsAddrPath)
}
cell := parts[2]
keyspace := parts[5]
shard := parts[6]
tabletType := topo.TabletType(parts[7])
if tabletType == "action" || tabletType == "actionlog" {
return nil, nil
}
addrs, _, err := wr.ts.GetEndPoints(ctx, cell, keyspace, shard, tabletType)
if err != nil {
return nil, err
}
// Write the individual endpoints and compute the SRV entries.
vtoccAddrs := LegacyZknsAddrs{make([]string, 0, 8)}
defaultAddrs := LegacyZknsAddrs{make([]string, 0, 8)}
for i, entry := range addrs.Entries {
zknsAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, i)
zknsPaths = append(zknsPaths, zknsAddrPath)
zknsAddr := zkns.ZknsAddr{
Host: entry.Host,
PortMap: entry.PortMap,
}
err := writeAddr(zconn, zknsAddrPath, &zknsAddr)
if err != nil {
return nil, err
}
defaultAddrs.Endpoints = append(defaultAddrs.Endpoints, zknsAddrPath)
vtoccAddrs.Endpoints = append(vtoccAddrs.Endpoints, zknsAddrPath+":vt")
}
// Prune any zkns entries that are no longer referenced by the
// shard graph.
deleteIdx := len(addrs.Entries)
for {
zknsStaleAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, deleteIdx)
// A "delete" is a write of sorts - just communicate up that nothing
// needs to be done to this node.
zknsPaths = append(zknsPaths, zknsStaleAddrPath)
err := zconn.Delete(zknsStaleAddrPath, -1)
if zookeeper.IsError(err, zookeeper.ZNONODE) {
break
}
if err != nil {
return nil, err
}
deleteIdx++
}
// Write the VDNS entries for both vtocc and mysql
vtoccVdnsPath := fmt.Sprintf("%v/vt.vdns", zknsAddrPath)
zknsPaths = append(zknsPaths, vtoccVdnsPath)
if err = writeAddrs(zconn, vtoccVdnsPath, &vtoccAddrs); err != nil {
return nil, err
}
defaultVdnsPath := fmt.Sprintf("%v.vdns", zknsAddrPath)
zknsPaths = append(zknsPaths, defaultVdnsPath)
if err = writeAddrs(zconn, defaultVdnsPath, &defaultAddrs); err != nil {
return nil, err
}
return zknsPaths, nil
}
// InitTablet initializes the tablet record if necessary.
func (agent *ActionAgent) InitTablet(port, securePort, gRPCPort int) error {
// only enabled if one of init_tablet_type (when healthcheck
// is disabled) or init_keyspace (when healthcheck is enabled)
// is passed in, then check other parameters
if *initTabletType == "" && *initKeyspace == "" {
return nil
}
// figure out our default target type
var tabletType topo.TabletType
if *initTabletType != "" {
if *targetTabletType != "" {
log.Fatalf("cannot specify both target_tablet_type and init_tablet_type parameters (as they might conflict)")
}
// use the type specified on the command line
tabletType = topo.TabletType(*initTabletType)
if !topo.IsTypeInList(tabletType, topo.AllTabletTypes) {
log.Fatalf("InitTablet encountered unknown init_tablet_type '%v'", *initTabletType)
}
if tabletType == topo.TYPE_MASTER || tabletType == topo.TYPE_SCRAP {
// We disallow TYPE_MASTER, so we don't have to change
// shard.MasterAlias, and deal with the corner cases.
// We also disallow TYPE_SCRAP, obviously.
log.Fatalf("init_tablet_type cannot be %v", tabletType)
}
} else if *targetTabletType != "" {
if tabletType := topo.TabletType(*targetTabletType); tabletType == topo.TYPE_MASTER {
log.Fatalf("target_tablet_type cannot be '%v'. Use '%v' instead.", tabletType, topo.TYPE_REPLICA)
}
// use spare, the healthcheck will turn us into what
// we need to be eventually
tabletType = topo.TYPE_SPARE
} else {
log.Fatalf("if init tablet is enabled, one of init_tablet_type or target_tablet_type needs to be specified")
}
// create a context for this whole operation
ctx, cancel := context.WithTimeout(agent.batchCtx, *initTimeout)
defer cancel()
// if we're assigned to a shard, make sure it exists, see if
// we are its master, and update its cells list if necessary
if tabletType != topo.TYPE_IDLE {
if *initKeyspace == "" || *initShard == "" {
log.Fatalf("if init tablet is enabled and the target type is not idle, init_keyspace and init_shard also need to be specified")
}
shard, _, err := topo.ValidateShardName(*initShard)
if err != nil {
log.Fatalf("cannot validate shard name: %v", err)
}
log.Infof("Reading shard record %v/%v", *initKeyspace, shard)
// read the shard, create it if necessary
si, err := topotools.GetOrCreateShard(ctx, agent.TopoServer, *initKeyspace, shard)
if err != nil {
return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err)
}
if si.MasterAlias == agent.TabletAlias {
// we are the current master for this shard (probably
// means the master tablet process was just restarted),
// so InitTablet as master.
tabletType = topo.TYPE_MASTER
}
// See if we need to add the tablet's cell to the shard's cell
// list. If we do, it has to be under the shard lock.
if !si.HasCell(agent.TabletAlias.Cell) {
actionNode := actionnode.UpdateShard()
lockPath, err := actionNode.LockShard(ctx, agent.TopoServer, *initKeyspace, shard)
if err != nil {
return fmt.Errorf("LockShard(%v/%v) failed: %v", *initKeyspace, shard, err)
}
// re-read the shard with the lock
si, err = agent.TopoServer.GetShard(ctx, *initKeyspace, shard)
if err != nil {
return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
}
// see if we really need to update it now
if !si.HasCell(agent.TabletAlias.Cell) {
si.Cells = append(si.Cells, agent.TabletAlias.Cell)
// write it back
if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil {
return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
}
}
// and unlock
if err := actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, nil); err != nil {
return err
}
}
}
log.Infof("Initializing the tablet for type %v", tabletType)
// figure out the hostname
hostname := *tabletHostname
if hostname == "" {
var err error
hostname, err = netutil.FullyQualifiedHostname()
if err != nil {
return err
}
}
// create and populate tablet record
tablet := &topo.Tablet{
Alias: agent.TabletAlias,
Hostname: hostname,
Portmap: make(map[string]int),
Keyspace: *initKeyspace,
Shard: *initShard,
Type: tabletType,
DbNameOverride: *initDbNameOverride,
Tags: initTags,
}
if port != 0 {
tablet.Portmap["vt"] = port
}
if securePort != 0 {
tablet.Portmap["vts"] = securePort
}
if gRPCPort != 0 {
tablet.Portmap["grpc"] = gRPCPort
}
if err := tablet.Complete(); err != nil {
return fmt.Errorf("InitTablet tablet.Complete failed: %v", err)
}
// now try to create the record
err := topo.CreateTablet(ctx, agent.TopoServer, tablet)
switch err {
case nil:
// it worked, we're good, can update the replication graph
if tablet.IsInReplicationGraph() {
if err := topo.UpdateTabletReplicationData(ctx, agent.TopoServer, tablet); err != nil {
return fmt.Errorf("UpdateTabletReplicationData failed: %v", err)
}
}
case topo.ErrNodeExists:
// The node already exists, will just try to update
// it. So we read it first.
oldTablet, err := agent.TopoServer.GetTablet(ctx, tablet.Alias)
if err != nil {
fmt.Errorf("InitTablet failed to read existing tablet record: %v", err)
}
// Sanity check the keyspace and shard
if oldTablet.Keyspace != tablet.Keyspace || oldTablet.Shard != tablet.Shard {
return fmt.Errorf("InitTablet failed because existing tablet keyspace and shard %v/%v differ from the provided ones %v/%v", oldTablet.Keyspace, oldTablet.Shard, tablet.Keyspace, tablet.Shard)
}
// And overwrite the rest
*(oldTablet.Tablet) = *tablet
if err := topo.UpdateTablet(ctx, agent.TopoServer, oldTablet); err != nil {
return fmt.Errorf("UpdateTablet failed: %v", err)
}
// Note we don't need to UpdateTabletReplicationData
// as the tablet already existed with the right data
// in the replication graph
default:
return fmt.Errorf("CreateTablet failed: %v", err)
}
// and now update the serving graph. Note we do that in any case,
// to clean any inaccurate record from any part of the serving graph.
if tabletType != topo.TYPE_IDLE {
if err := topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, tablet); err != nil {
return fmt.Errorf("UpdateTabletEndpoints failed: %v", err)
}
}
return nil
}
func main() {
flag.Parse()
servenv.Init()
defer servenv.Close()
templateLoader = NewTemplateLoader(*templateDir, dummyTemplate, *debug)
ts = topo.GetServer()
defer topo.CloseServers()
wr := wrangler.New(logutil.NewConsoleLogger(), ts, 30*time.Second, 30*time.Second)
actionRepo = NewActionRepository(ts)
// keyspace actions
actionRepo.RegisterKeyspaceAction("ValidateKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateKeyspace(keyspace, false)
})
actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateSchemaKeyspace(keyspace, nil, false)
})
actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateVersionKeyspace(keyspace)
})
actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidatePermissionsKeyspace(keyspace)
})
// shard actions
actionRepo.RegisterShardAction("ValidateShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateShard(keyspace, shard, false)
})
actionRepo.RegisterShardAction("ValidateSchemaShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateSchemaShard(keyspace, shard, nil, false)
})
actionRepo.RegisterShardAction("ValidateVersionShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateVersionShard(keyspace, shard)
})
actionRepo.RegisterShardAction("ValidatePermissionsShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidatePermissionsShard(keyspace, shard)
})
// tablet actions
actionRepo.RegisterTabletAction("RpcPing", "",
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
return "", wr.ActionInitiator().RpcPing(tabletAlias, 10*time.Second)
})
actionRepo.RegisterTabletAction("ScrapTablet", acl.ADMIN,
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
// refuse to scrap tablets that are not spare
ti, err := wr.TopoServer().GetTablet(tabletAlias)
if err != nil {
return "", err
}
if ti.Type != topo.TYPE_SPARE {
return "", fmt.Errorf("Can only scrap spare tablets")
}
actionPath, err := wr.Scrap(tabletAlias, false, false)
if err != nil {
return "", err
}
return "", wr.WaitForCompletion(actionPath)
})
actionRepo.RegisterTabletAction("ScrapTabletForce", acl.ADMIN,
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
// refuse to scrap tablets that are not spare
ti, err := wr.TopoServer().GetTablet(tabletAlias)
if err != nil {
return "", err
}
if ti.Type != topo.TYPE_SPARE {
return "", fmt.Errorf("Can only scrap spare tablets")
}
_, err = wr.Scrap(tabletAlias, true, false)
return "", err
})
actionRepo.RegisterTabletAction("DeleteTablet", acl.ADMIN,
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
return "", wr.DeleteTablet(tabletAlias)
})
// toplevel index
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
templateLoader.ServeTemplate("index.html", indexContent, w, r)
})
// keyspace actions
http.HandleFunc("/keyspace_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "no keyspace provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyKeyspaceAction(action, keyspace, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
// shard actions
http.HandleFunc("/shard_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "no keyspace provided", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "no shard provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyShardAction(action, keyspace, shard, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
// tablet actions
http.HandleFunc("/tablet_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
alias := r.FormValue("alias")
if alias == "" {
http.Error(w, "no alias provided", http.StatusBadRequest)
return
}
tabletAlias, err := topo.ParseTabletAliasString(alias)
if err != nil {
http.Error(w, "bad alias provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyTabletAction(action, tabletAlias, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
// topology server
http.HandleFunc("/dbtopo", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
result := DbTopologyResult{}
topology, err := topotools.DbTopology(wr.TopoServer())
if err != nil {
result.Error = err.Error()
} else {
result.Topology = topology
}
templateLoader.ServeTemplate("dbtopo.html", result, w, r)
})
// serving graph
http.HandleFunc("/serving_graph/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(r.URL.Path, "/")
cell := parts[len(parts)-1]
if cell == "" {
cells, err := ts.GetKnownCells()
if err != nil {
httpError(w, "cannot get known cells: %v", err)
return
} else {
templateLoader.ServeTemplate("serving_graph_cells.html", cells, w, r)
}
return
}
servingGraph := topotools.DbServingGraph(wr.TopoServer(), cell)
templateLoader.ServeTemplate("serving_graph.html", servingGraph, w, r)
})
// redirects for explorers
http.HandleFunc("/explorers/redirect", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
var explorer Explorer
switch len(explorers) {
case 0:
http.Error(w, "no explorer configured", http.StatusInternalServerError)
return
case 1:
for _, ex := range explorers {
explorer = ex
}
default:
explorerName := r.FormValue("explorer")
var ok bool
explorer, ok = explorers[explorerName]
if !ok {
http.Error(w, "bad explorer name", http.StatusBadRequest)
return
}
}
var target string
switch r.FormValue("type") {
case "keyspace":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetKeyspacePath(keyspace)
case "shard":
keyspace, shard := r.FormValue("keyspace"), r.FormValue("shard")
if keyspace == "" || shard == "" {
http.Error(w, "keyspace and shard are obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetShardPath(keyspace, shard)
case "srv_keyspace":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvKeyspacePath(cell, keyspace)
case "srv_shard":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvShardPath(cell, keyspace, shard)
case "srv_type":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
tabletType := r.FormValue("tablet_type")
if tabletType == "" {
http.Error(w, "tablet_type is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType))
case "tablet":
aliasName := r.FormValue("alias")
if aliasName == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
alias, err := topo.ParseTabletAliasString(aliasName)
if err != nil {
http.Error(w, "bad tablet alias", http.StatusBadRequest)
return
}
target = explorer.GetTabletPath(alias)
case "replication":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetReplicationSlaves(cell, keyspace, shard)
default:
http.Error(w, "bad redirect type", http.StatusBadRequest)
return
}
http.Redirect(w, r, target, http.StatusFound)
})
servenv.RunDefault()
}
func main() {
flag.Parse()
servenv.Init()
defer servenv.Close()
templateLoader = NewTemplateLoader(*templateDir, dummyTemplate, *debug)
ts := topo.GetServer()
defer topo.CloseServers()
wr := wrangler.New(ts, 30*time.Second, 30*time.Second)
actionRepo = NewActionRepository(wr)
// keyspace actions
actionRepo.RegisterKeyspaceAction("ValidateKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateKeyspace(keyspace, false)
})
actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateSchemaKeyspace(keyspace, false)
})
actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateVersionKeyspace(keyspace)
})
actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidatePermissionsKeyspace(keyspace)
})
// shard actions
actionRepo.RegisterShardAction("ValidateShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateShard(keyspace, shard, false)
})
actionRepo.RegisterShardAction("ValidateSchemaShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateSchemaShard(keyspace, shard, false)
})
actionRepo.RegisterShardAction("ValidateVersionShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateVersionShard(keyspace, shard)
})
actionRepo.RegisterShardAction("ValidatePermissionsShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidatePermissionsShard(keyspace, shard)
})
// tablet actions
actionRepo.RegisterTabletAction("RpcPing",
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
return "", wr.ActionInitiator().RpcPing(tabletAlias, 10*time.Second)
})
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
templateLoader.ServeTemplate("index.html", indexContent, w, r)
})
http.HandleFunc("/keyspace_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "no keyspace provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyKeyspaceAction(action, keyspace, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
http.HandleFunc("/shard_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "no keyspace provided", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "no shard provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyShardAction(action, keyspace, shard, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
http.HandleFunc("/tablet_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
alias := r.FormValue("alias")
if alias == "" {
http.Error(w, "no alias provided", http.StatusBadRequest)
return
}
tabletAlias, err := topo.ParseTabletAliasString(alias)
if err != nil {
http.Error(w, "bad alias provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyTabletAction(action, tabletAlias, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
http.HandleFunc("/dbtopo", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
result := DbTopologyResult{}
topology, err := wr.DbTopology()
if err != nil {
result.Error = err.Error()
} else {
result.Topology = topology
}
templateLoader.ServeTemplate("dbtopo.html", result, w, r)
})
http.HandleFunc("/serving_graph/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(r.URL.Path, "/")
cell := parts[len(parts)-1]
if cell == "" {
cells, err := wr.TopoServer().GetKnownCells()
if err != nil {
httpError(w, "cannot get known cells: %v", err)
return
} else {
templateLoader.ServeTemplate("serving_graph_cells.html", cells, w, r)
}
return
}
result := ServingGraphResult{}
servingGraph, err := wr.ServingGraph(cell)
if err != nil {
result.Error = err.Error()
} else {
result.ServingGraph = servingGraph
}
templateLoader.ServeTemplate("serving_graph.html", result, w, r)
})
http.HandleFunc("/explorers/redirect", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
explorerName := r.FormValue("explorer")
explorer, ok := explorers[explorerName]
if !ok {
http.Error(w, "bad explorer name", http.StatusBadRequest)
return
}
var target string
switch r.FormValue("type") {
case "keyspace":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetKeyspacePath(keyspace)
case "shard":
keyspace, shard := r.FormValue("keyspace"), r.FormValue("shard")
if keyspace == "" || shard == "" {
http.Error(w, "keyspace and shard are obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetShardPath(keyspace, shard)
case "srv_keyspace":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvKeyspacePath(cell, keyspace)
case "srv_shard":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvShardPath(cell, keyspace, shard)
case "srv_type":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
tabletType := r.FormValue("tablet_type")
if tabletType == "" {
http.Error(w, "tablet_type is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType))
case "tablet":
aliasName := r.FormValue("alias")
if aliasName == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
alias, err := topo.ParseTabletAliasString(aliasName)
if err != nil {
http.Error(w, "bad tablet alias", http.StatusBadRequest)
return
}
target = explorer.GetTabletPath(alias)
case "replication":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetReplicationSlaves(cell, keyspace, shard)
default:
http.Error(w, "bad redirect type", http.StatusBadRequest)
return
}
http.Redirect(w, r, target, http.StatusFound)
})
servenv.Run(*port)
}
// This function should only be used with an action lock on the keyspace
// - otherwise the consistency of the serving graph data can't be
// guaranteed.
//
// Take data from the global keyspace and rebuild the local serving
// copies in each cell.
func (wr *Wrangler) rebuildKeyspace(keyspace string, cells []string) error {
relog.Info("rebuildKeyspace %v", keyspace)
shards, err := wr.ts.GetShardNames(keyspace)
if err != nil {
return err
}
// Rebuild all shards in parallel.
wg := sync.WaitGroup{}
er := concurrency.FirstErrorRecorder{}
for _, shard := range shards {
wg.Add(1)
go func(shard string) {
if err := wr.RebuildShardGraph(keyspace, shard, cells); err != nil {
er.RecordError(fmt.Errorf("RebuildShardGraph failed: %v/%v %v", keyspace, shard, err))
}
wg.Done()
}(shard)
}
wg.Wait()
if er.HasErrors() {
return er.Error()
}
// Scan the first shard to discover which cells need local serving data.
aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, keyspace, shards[0])
if err != nil {
return err
}
// srvKeyspaceByPath is a map:
// key: local keyspace {cell,keyspace}
// value: topo.SrvKeyspace object being built
srvKeyspaceByPath := make(map[cellKeyspace]*topo.SrvKeyspace)
for _, alias := range aliases {
keyspaceLocation := cellKeyspace{alias.Cell, keyspace}
if _, ok := srvKeyspaceByPath[keyspaceLocation]; !ok {
// before adding keyspaceLocation to the map of
// of KeyspaceByPath, we check this is a
// serving tablet. No serving tablet in shard
// 0 means we're not rebuilding the serving
// graph in that cell. This is somewhat
// expensive, but we only do it on all the
// non-serving tablets in a shard before we
// find a serving tablet.
ti, err := wr.ts.GetTablet(alias)
if err != nil {
return err
}
if !ti.IsServingType() {
continue
}
srvKeyspaceByPath[keyspaceLocation] = &topo.SrvKeyspace{Shards: make([]topo.SrvShard, 0, 16)}
}
}
// for each entry in the srvKeyspaceByPath map, we do the following:
// - read the ShardInfo structures for each shard
// - prune the AddrsByType field, result would be too big
// - compute the union of the db types (replica, master, ...)
// - sort the shards in the list by range
// - check the ranges are compatible (no hole, covers everything)
for srvPath, srvKeyspace := range srvKeyspaceByPath {
keyspaceDbTypes := make(map[topo.TabletType]bool)
for _, shard := range shards {
srvShard, err := wr.ts.GetSrvShard(srvPath.cell, srvPath.keyspace, shard)
if err != nil {
return err
}
for dbType, _ := range srvShard.AddrsByType {
keyspaceDbTypes[topo.TabletType(dbType)] = true
}
// Prune addrs, this is unnecessarily expensive right now. It is easier to
// load on-demand since we have to do that anyway on a reconnect.
srvShard.AddrsByType = nil
srvKeyspace.Shards = append(srvKeyspace.Shards, *srvShard)
}
tabletTypes := make([]topo.TabletType, 0, len(keyspaceDbTypes))
for dbType, _ := range keyspaceDbTypes {
tabletTypes = append(tabletTypes, dbType)
}
srvKeyspace.TabletTypes = tabletTypes
// FIXME(msolomon) currently this only works when the shards are range-based
topo.SrvShardArray(srvKeyspace.Shards).Sort()
// check the first Start is MinKey, the last End is MaxKey,
// and the values in between match: End[i] == Start[i+1]
if srvKeyspace.Shards[0].KeyRange.Start != key.MinKey {
return fmt.Errorf("Keyspace does not start with %v", key.MinKey)
}
if srvKeyspace.Shards[len(srvKeyspace.Shards)-1].KeyRange.End != key.MaxKey {
return fmt.Errorf("Keyspace does not end with %v", key.MaxKey)
}
for i, _ := range srvKeyspace.Shards[0 : len(srvKeyspace.Shards)-1] {
if srvKeyspace.Shards[i].KeyRange.End != srvKeyspace.Shards[i+1].KeyRange.Start {
return fmt.Errorf("Non-contiguous KeyRange values at shard %v to %v: %v != %v", i, i+1, srvKeyspace.Shards[i].KeyRange.End.Hex(), srvKeyspace.Shards[i+1].KeyRange.Start.Hex())
}
}
}
// and then finally save the keyspace objects
for srvPath, srvKeyspace := range srvKeyspaceByPath {
if err := wr.ts.UpdateSrvKeyspace(srvPath.cell, srvPath.keyspace, srvKeyspace); err != nil {
return fmt.Errorf("writing serving data failed: %v", err)
}
}
return nil
}