func TestQueryShard(t *testing.T) { reflected, err := bson.Marshal(&reflectQueryShard{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, Keyspace: "keyspace", Shards: []string{"shard1", "shard2"}, TabletType: topo.TabletType("replica"), Session: &commonSession, }) if err != nil { t.Error(err) } want := string(reflected) custom := QueryShard{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, Keyspace: "keyspace", Shards: []string{"shard1", "shard2"}, TabletType: topo.TabletType("replica"), Session: &commonSession, } encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%#v, got\n%#v", want, got) } var unmarshalled QueryShard err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if !reflect.DeepEqual(custom, unmarshalled) { t.Errorf("want \n%#v, got \n%#v", custom, unmarshalled) } unexpected, err := bson.Marshal(&badQueryShard{}) if err != nil { t.Error(err) } err = bson.Unmarshal(unexpected, &unmarshalled) want = "Unrecognized tag Extra" if err == nil || want != err.Error() { t.Errorf("want %v, got %v", want, err) } }
func TestSession(t *testing.T) { reflected, err := bson.Marshal(&reflectSession{ InTransaction: true, ShardSessions: []*ShardSession{{ Keyspace: "a", Shard: "0", TabletType: topo.TabletType("replica"), TransactionId: 1, }, { Keyspace: "b", Shard: "1", TabletType: topo.TabletType("master"), TransactionId: 2, }}, }) if err != nil { t.Error(err) } want := string(reflected) custom := commonSession encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%#v, got\n%#v", want, got) } var unmarshalled Session err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if !reflect.DeepEqual(custom, unmarshalled) { t.Errorf("want \n%#v, got \n%#v", custom, unmarshalled) } unexpected, err := bson.Marshal(&badSession{}) if err != nil { t.Error(err) } err = bson.Unmarshal(unexpected, &unmarshalled) want = "Unrecognized tag Extra" if err == nil || want != err.Error() { t.Errorf("want %v, got %v", want, err) } }
func getSrvKeyspace(rpcClient *rpcplus.Client, cell, keyspace string, verbose bool) { req := &topo.GetSrvKeyspaceArgs{ Cell: cell, Keyspace: keyspace, } reply := &topo.SrvKeyspace{} if err := rpcClient.Call("TopoReader.GetSrvKeyspace", req, reply); err != nil { log.Fatalf("TopoReader.GetSrvKeyspace error: %v", err) } if verbose { tabletTypes := make([]string, 0, len(reply.Partitions)) for t, _ := range reply.Partitions { tabletTypes = append(tabletTypes, string(t)) } sort.Strings(tabletTypes) for _, t := range tabletTypes { println(fmt.Sprintf("Partitions[%v] =", t)) for i, s := range reply.Partitions[topo.TabletType(t)].Shards { println(fmt.Sprintf(" Shards[%v]=%v", i, s.KeyRange.String())) } } for i, s := range reply.Shards { println(fmt.Sprintf("Shards[%v]=%v", i, s.KeyRange.String())) } for i, t := range reply.TabletTypes { println(fmt.Sprintf("TabletTypes[%v] = %v", i, t)) } } }
// UnmarshalBson unmarshals QueryShard from buf. func (qrs *QueryShard) UnmarshalBson(buf *bytes.Buffer) { bson.Next(buf, 4) kind := bson.NextByte(buf) for kind != bson.EOO { key := bson.ReadCString(buf) switch key { case "Sql": qrs.Sql = bson.DecodeString(buf, kind) case "BindVariables": qrs.BindVariables = tproto.DecodeBindVariablesBson(buf, kind) case "Keyspace": qrs.Keyspace = bson.DecodeString(buf, kind) case "TabletType": qrs.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "Shards": qrs.Shards = bson.DecodeStringArray(buf, kind) case "Session": qrs.Session = new(Session) qrs.Session.UnmarshalBson(buf) default: panic(bson.NewBsonError("Unrecognized tag %s", key)) } kind = bson.NextByte(buf) } }
func (sqs *StreamQueryKeyRange) UnmarshalBson(buf *bytes.Buffer) { bson.Next(buf, 4) kind := bson.NextByte(buf) for kind != bson.EOO { keyName := bson.ReadCString(buf) switch keyName { case "Sql": sqs.Sql = bson.DecodeString(buf, kind) case "BindVariables": sqs.BindVariables = tproto.DecodeBindVariablesBson(buf, kind) case "Keyspace": sqs.Keyspace = bson.DecodeString(buf, kind) case "KeyRange": sqs.KeyRange = bson.DecodeString(buf, kind) case "TabletType": sqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "Session": sqs.Session = new(Session) sqs.Session.UnmarshalBson(buf) default: panic(bson.NewBsonError("Unrecognized tag %s", keyName)) } kind = bson.NextByte(buf) } }
// for direct zk connection: vtzk://host:port/cell/keyspace/tabletType // we always use a MetaConn, host and port are ignored. // the driver name dictates if we use zk or zkocc, and streaming or not // // if user and password are specified in the URL, they will be used // for each DB connection to the tablet's vttablet processes func (driver *sDriver) Open(name string) (sc db.Conn, err error) { if !strings.HasPrefix(name, "vtzk://") { // add a default protocol talking to zk name = "vtzk://" + name } u, err := url.Parse(name) if err != nil { return nil, err } dbi, tabletType := path.Split(u.Path) dbi = strings.Trim(dbi, "/") tabletType = strings.Trim(tabletType, "/") cell, keyspace := path.Split(dbi) cell = strings.Trim(cell, "/") keyspace = strings.Trim(keyspace, "/") var user, password string if u.User != nil { user = u.User.Username() var ok bool password, ok = u.User.Password() if !ok { return nil, fmt.Errorf("vt: need a password if a user is specified") } } return Dial(driver.ts, cell, keyspace, topo.TabletType(tabletType), driver.stream, tablet.DefaultTimeout, user, password) }
// UnmarshalBson unmarshals BatchQueryShard from buf. func (bqs *BatchQueryShard) UnmarshalBson(buf *bytes.Buffer, kind byte) { bson.VerifyObject(kind) bson.Next(buf, 4) kind = bson.NextByte(buf) for kind != bson.EOO { keyName := bson.ReadCString(buf) switch keyName { case "Queries": bqs.Queries = tproto.DecodeQueriesBson(buf, kind) case "Keyspace": bqs.Keyspace = bson.DecodeString(buf, kind) case "Shards": bqs.Shards = bson.DecodeStringArray(buf, kind) case "TabletType": bqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "Session": if kind != bson.Null { bqs.Session = new(Session) bqs.Session.UnmarshalBson(buf, kind) } default: bson.Skip(buf, kind) } kind = bson.NextByte(buf) } }
func (sqs *StreamQueryKeyRange) UnmarshalBson(buf *bytes.Buffer, kind byte) { bson.VerifyObject(kind) bson.Next(buf, 4) kind = bson.NextByte(buf) for kind != bson.EOO { keyName := bson.ReadCString(buf) switch keyName { case "Sql": sqs.Sql = bson.DecodeString(buf, kind) case "BindVariables": sqs.BindVariables = tproto.DecodeBindVariablesBson(buf, kind) case "Keyspace": sqs.Keyspace = bson.DecodeString(buf, kind) case "KeyRange": sqs.KeyRange = bson.DecodeString(buf, kind) case "TabletType": sqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "Session": if kind != bson.Null { sqs.Session = new(Session) sqs.Session.UnmarshalBson(buf, kind) } default: bson.Skip(buf, kind) } kind = bson.NextByte(buf) } }
func TestHashResolve(t *testing.T) { hind := NewHashIndex(TEST_SHARDED, new(sandboxTopo), "") nn, _ := sqltypes.BuildNumeric("11") ks, shards, err := hind.Resolve(topo.TabletType("master"), []interface{}{1, int32(2), int64(3), uint(4), uint32(5), uint64(6), nn}) if err != nil { t.Error(err) } want := []string{"-20", "-20", "40-60", "c0-e0", "60-80", "e0-", "a0-c0"} if !reflect.DeepEqual(shards, want) { t.Errorf("got\n%#v, want\n%#v", shards, want) } if ks != TEST_SHARDED { t.Errorf("got %v, want TEST_SHARDED", ks) } _, _, err = hind.Resolve(topo.TabletType("master"), []interface{}{"aa"}) wantErr := "unexpected type for aa: string" if err == nil || err.Error() != wantErr { t.Errorf("got %v, want %v", err, wantErr) } }
func (agent *ActionAgent) initHeathCheck() { if !agent.IsRunningHealthCheck() { log.Infof("No target_tablet_type specified, disabling any health check") return } log.Infof("Starting periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType) t := timer.NewTimer(*healthCheckInterval) servenv.OnTerm(func() { // When we enter lameduck mode, we want to not call // the health check any more. After this returns, we // are guaranteed to not call it. log.Info("Stopping periodic health check timer") t.Stop() // Now we can finish up and force ourselves to not healthy. agent.terminateHealthChecks(topo.TabletType(*targetTabletType)) }) t.Start(func() { agent.runHealthCheck(topo.TabletType(*targetTabletType)) }) }
// handleExplorerRedirect returns the redirect target URL. func handleExplorerRedirect(r *http.Request) (string, error) { keyspace := r.FormValue("keyspace") shard := r.FormValue("shard") cell := r.FormValue("cell") switch r.FormValue("type") { case "keyspace": if keyspace == "" { return "", errors.New("keyspace is required for this redirect") } return explorer.GetKeyspacePath(keyspace), nil case "shard": if keyspace == "" || shard == "" { return "", errors.New("keyspace and shard are required for this redirect") } return explorer.GetShardPath(keyspace, shard), nil case "srv_keyspace": if keyspace == "" || cell == "" { return "", errors.New("keyspace and cell are required for this redirect") } return explorer.GetSrvKeyspacePath(cell, keyspace), nil case "srv_shard": if keyspace == "" || shard == "" || cell == "" { return "", errors.New("keyspace, shard, and cell are required for this redirect") } return explorer.GetSrvShardPath(cell, keyspace, shard), nil case "srv_type": tabletType := r.FormValue("tablet_type") if keyspace == "" || shard == "" || cell == "" || tabletType == "" { return "", errors.New("keyspace, shard, cell, and tablet_type are required for this redirect") } return explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType)), nil case "tablet": alias := r.FormValue("alias") if alias == "" { return "", errors.New("alias is required for this redirect") } tabletAlias, err := topo.ParseTabletAliasString(alias) if err != nil { return "", fmt.Errorf("bad tablet alias %q: %v", alias, err) } return explorer.GetTabletPath(tabletAlias), nil case "replication": if keyspace == "" || shard == "" || cell == "" { return "", errors.New("keyspace, shard, and cell are required for this redirect") } return explorer.GetReplicationSlaves(cell, keyspace, shard), nil default: return "", errors.New("bad redirect type") } }
func initHeathCheck(agent *tabletmanager.ActionAgent) { if *targetTabletType == "" { log.Infof("No target_tablet_type specified, disabling any health check") return } log.Infof("Starting up periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType) go func() { t := time.NewTicker(*healthCheckInterval) for _ = range t.C { agent.RunHealthCheck(topo.TabletType(*targetTabletType)) } }() }
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) { zkSgShardPath := zkPathForVtShard(cell, keyspace, shard) children, _, err := zkts.zconn.Children(zkSgShardPath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } return nil, err } result := make([]topo.TabletType, len(children)) for i, tt := range children { result[i] = topo.TabletType(tt) } return result, nil }
func (spm *SessionParams) UnmarshalBson(buf *bytes.Buffer) { bson.Next(buf, 4) kind := bson.NextByte(buf) for kind != bson.EOO { key := bson.ReadCString(buf) switch key { case "TabletType": spm.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) default: panic(bson.NewBsonError("Unrecognized tag %s", key)) } kind = bson.NextByte(buf) } }
func TestSessionParams(t *testing.T) { reflected, err := bson.Marshal(&reflectSessionParams{topo.TabletType("master")}) if err != nil { t.Error(err) } want := string(reflected) custom := SessionParams{topo.TabletType("master")} encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%#v, got\n%#v", want, got) } var unmarshalled SessionParams err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if custom != unmarshalled { t.Errorf("want %v, got %#v", custom, unmarshalled) } unexpected, err := bson.Marshal(&badSessionParams{}) if err != nil { t.Error(err) } err = bson.Unmarshal(unexpected, &unmarshalled) want = "Unrecognized tag Extra" if err == nil || want != err.Error() { t.Errorf("want %v, got %v", want, err) } }
func getEndPoints(rpcClient *rpcplus.Client, cell, keyspace, shard, tabletType string, verbose bool) { req := &topo.GetEndPointsArgs{ Cell: cell, Keyspace: keyspace, Shard: shard, TabletType: topo.TabletType(tabletType), } reply := &topo.EndPoints{} if err := rpcClient.Call("TopoReader.GetEndPoints", req, reply); err != nil { log.Fatalf("TopoReader.GetEndPoints error: %v", err) } if verbose { for i, e := range reply.Entries { println(fmt.Sprintf("Entries[%v] = %v %v", i, e.Uid, e.Host)) } } }
// for direct zk connection: vtzk://host:port/cell/keyspace/tabletType // we always use a MetaConn, host and port are ignored. // the driver name dictates if we use zk or zkocc, and streaming or not func (driver *sDriver) Open(name string) (sc db.Conn, err error) { if !strings.HasPrefix(name, "vtzk://") { // add a default protocol talking to zk name = "vtzk://" + name } u, err := url.Parse(name) if err != nil { return nil, err } dbi, tabletType := path.Split(u.Path) dbi = strings.Trim(dbi, "/") tabletType = strings.Trim(tabletType, "/") cell, keyspace := path.Split(dbi) cell = strings.Trim(cell, "/") keyspace = strings.Trim(keyspace, "/") return Dial(driver.ts, cell, keyspace, topo.TabletType(tabletType), driver.stream, tablet.DefaultTimeout) }
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) { zkSgShardPath := zkPathForVtShard(cell, keyspace, shard) children, _, err := zkts.zconn.Children(zkSgShardPath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } return nil, err } result := make([]topo.TabletType, 0, len(children)) for _, tt := range children { // these two are used for locking if tt == "action" || tt == "actionlog" { continue } result = append(result, topo.TabletType(tt)) } return result, nil }
// GetSrvTabletTypesPerShard implements topo.Server. func (s *Server) GetSrvTabletTypesPerShard(ctx context.Context, cellName, keyspace, shard string) ([]topo.TabletType, error) { cell, err := s.getCell(cellName) if err != nil { return nil, err } resp, err := cell.Get(srvShardDirPath(keyspace, shard), false /* sort */, false /* recursive */) if err != nil { return nil, convertError(err) } if resp.Node == nil { return nil, ErrBadResponse } tabletTypes := make([]topo.TabletType, 0, len(resp.Node.Nodes)) for _, n := range resp.Node.Nodes { tabletTypes = append(tabletTypes, topo.TabletType(path.Base(n.Key))) } return tabletTypes, nil }
// UnmarshalBson unmarshals ShardSession from buf. func (shardSession *ShardSession) UnmarshalBson(buf *bytes.Buffer) { bson.Next(buf, 4) kind := bson.NextByte(buf) for kind != bson.EOO { key := bson.ReadCString(buf) switch key { case "Keyspace": shardSession.Keyspace = bson.DecodeString(buf, kind) case "Shard": shardSession.Shard = bson.DecodeString(buf, kind) case "TabletType": shardSession.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "TransactionId": shardSession.TransactionId = bson.DecodeInt64(buf, kind) default: panic(bson.NewBsonError("Unrecognized tag %s", key)) } kind = bson.NextByte(buf) } }
// UnmarshalBson unmarshals ShardSession from buf. func (shardSession *ShardSession) UnmarshalBson(buf *bytes.Buffer, kind byte) { bson.VerifyObject(kind) bson.Next(buf, 4) kind = bson.NextByte(buf) for kind != bson.EOO { keyName := bson.ReadCString(buf) switch keyName { case "Keyspace": shardSession.Keyspace = bson.DecodeString(buf, kind) case "Shard": shardSession.Shard = bson.DecodeString(buf, kind) case "TabletType": shardSession.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "TransactionId": shardSession.TransactionId = bson.DecodeInt64(buf, kind) default: bson.Skip(buf, kind) } kind = bson.NextByte(buf) } }
// UnmarshalBson unmarshals BatchQueryShard from buf. func (bqs *BatchQueryShard) UnmarshalBson(buf *bytes.Buffer) { bson.Next(buf, 4) kind := bson.NextByte(buf) for kind != bson.EOO { keyName := bson.ReadCString(buf) switch keyName { case "Queries": bqs.Queries = tproto.DecodeQueriesBson(buf, kind) case "Keyspace": bqs.Keyspace = bson.DecodeString(buf, kind) case "Shards": bqs.Shards = bson.DecodeStringArray(buf, kind) case "TabletType": bqs.TabletType = topo.TabletType(bson.DecodeString(buf, kind)) case "Session": bqs.Session = new(Session) bqs.Session.UnmarshalBson(buf) default: panic(bson.NewBsonError("Unrecognized tag %s", keyName)) } kind = bson.NextByte(buf) } }
func initAPI(ctx context.Context, ts topo.Server, actions *ActionRepository) { tabletHealthCache := newTabletHealthCache(ts) // Cells handleCollection("cells", func(r *http.Request) (interface{}, error) { if getItemPath(r.URL.Path) != "" { return nil, errors.New("cells can only be listed, not retrieved") } return ts.GetKnownCells(ctx) }) // Keyspaces handleCollection("keyspaces", func(r *http.Request) (interface{}, error) { keyspace := getItemPath(r.URL.Path) // List all keyspaces. if keyspace == "" { return ts.GetKeyspaces(ctx) } // Perform an action on a keyspace. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyKeyspaceAction(ctx, action, keyspace, r), nil } // Get the keyspace record. return ts.GetKeyspace(ctx, keyspace) }) // Shards handleCollection("shards", func(r *http.Request) (interface{}, error) { shardPath := getItemPath(r.URL.Path) if !strings.Contains(shardPath, "/") { return nil, fmt.Errorf("invalid shard path: %q", shardPath) } parts := strings.SplitN(shardPath, "/", 2) keyspace := parts[0] shard := parts[1] // List the shards in a keyspace. if shard == "" { return ts.GetShardNames(ctx, keyspace) } // Perform an action on a shard. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyShardAction(ctx, action, keyspace, shard, r), nil } // Get the shard record. return ts.GetShard(ctx, keyspace, shard) }) // Tablets handleCollection("tablets", func(r *http.Request) (interface{}, error) { tabletPath := getItemPath(r.URL.Path) // List tablets based on query params. if tabletPath == "" { if err := r.ParseForm(); err != nil { return nil, err } shardRef := r.FormValue("shard") cell := r.FormValue("cell") if shardRef != "" { // Look up by keyspace/shard, and optionally cell. keyspace, shard, err := topo.ParseKeyspaceShardString(shardRef) if err != nil { return nil, err } if cell != "" { return topo.FindAllTabletAliasesInShardByCell(ctx, ts, keyspace, shard, []string{cell}) } return topo.FindAllTabletAliasesInShard(ctx, ts, keyspace, shard) } // Get all tablets in a cell. if cell == "" { return nil, errors.New("cell param required") } return ts.GetTabletsByCell(ctx, cell) } // Get tablet health. if parts := strings.Split(tabletPath, "/"); len(parts) == 2 && parts[1] == "health" { tabletAlias, err := topo.ParseTabletAliasString(parts[0]) if err != nil { return nil, err } return tabletHealthCache.Get(ctx, tabletAlias) } tabletAlias, err := topo.ParseTabletAliasString(tabletPath) if err != nil { return nil, err } // Perform an action on a tablet. if r.Method == "POST" { if err := r.ParseForm(); err != nil { return nil, err } action := r.FormValue("action") if action == "" { return nil, errors.New("must specify action") } return actions.ApplyTabletAction(ctx, action, tabletAlias, r), nil } // Get the tablet record. return ts.GetTablet(ctx, tabletAlias) }) // EndPoints handleCollection("endpoints", func(r *http.Request) (interface{}, error) { // We expect cell/keyspace/shard/tabletType. epPath := getItemPath(r.URL.Path) parts := strings.Split(epPath, "/") if len(parts) != 4 { return nil, fmt.Errorf("invalid cell/keyspace/shard/tabletType: %q", epPath) } if parts[3] == "" { // tabletType is empty, so list the tablet types. return ts.GetSrvTabletTypesPerShard(ctx, parts[0], parts[1], parts[2]) } // Get the endpoints object for a specific type. ep, _, err := ts.GetEndPoints(ctx, parts[0], parts[1], parts[2], topo.TabletType(parts[3])) return ep, err }) // Schema Change http.HandleFunc(apiPrefix+"schema/apply", func(w http.ResponseWriter, r *http.Request) { req := struct{ Keyspace, SQL string }{} if err := unmarshalRequest(r, &req); err != nil { httpErrorf(w, r, "can't unmarshal request: %v", err) return } executor := schemamanager.NewTabletExecutor( tmclient.NewTabletManagerClient(), ts) schemamanager.Run(ctx, schemamanager.NewUIController(req.SQL, req.Keyspace, w), executor) }) // VSchema http.HandleFunc(apiPrefix+"vschema/", func(w http.ResponseWriter, r *http.Request) { schemafier, ok := ts.(topo.Schemafier) if !ok { httpErrorf(w, r, "%T doesn't support schemafier API", ts) return } // Save VSchema if r.Method == "POST" { vschema, err := ioutil.ReadAll(r.Body) if err != nil { httpErrorf(w, r, "can't read request body: %v", err) return } if err := schemafier.SaveVSchema(ctx, string(vschema)); err != nil { httpErrorf(w, r, "can't save vschema: %v", err) } return } // Get VSchema vschema, err := schemafier.GetVSchema(ctx) if err != nil { httpErrorf(w, r, "can't get vschema: %v", err) return } w.Header().Set("Content-Type", jsonContentType) w.Write([]byte(vschema)) }) }
func TestKeyspaceIdBatchQuery(t *testing.T) { reflected, err := bson.Marshal(&reflectKeyspaceIdBatchQuery{ Queries: []reflectBoundKeyspaceIdQuery{{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, Keyspace: "keyspace", KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")}, }}, Session: &Session{InTransaction: true, ShardSessions: []*ShardSession{{ Keyspace: "a", Shard: "0", TabletType: topo.TabletType("replica"), TransactionId: 1, }, { Keyspace: "b", Shard: "1", TabletType: topo.TabletType("master"), TransactionId: 2, }}, }, }) if err != nil { t.Error(err) } want := string(reflected) custom := KeyspaceIdBatchQuery{ Queries: []BoundKeyspaceIdQuery{{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, Keyspace: "keyspace", KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")}, }}, Session: &commonSession, } encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%+v, got\n%+v", want, got) } var unmarshalled KeyspaceIdBatchQuery err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if !reflect.DeepEqual(custom, unmarshalled) { t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled) } extra, err := bson.Marshal(&extraKeyspaceIdBatchQuery{}) if err != nil { t.Error(err) } err = bson.Unmarshal(extra, &unmarshalled) if err != nil { t.Error(err) } }
"reflect" "testing" "github.com/youtube/vitess/go/bson" mproto "github.com/youtube/vitess/go/mysql/proto" "github.com/youtube/vitess/go/sqltypes" kproto "github.com/youtube/vitess/go/vt/key" "github.com/youtube/vitess/go/vt/topo" ) var commonSession = Session{ InTransaction: true, ShardSessions: []*ShardSession{{ Keyspace: "a", Shard: "0", TabletType: topo.TabletType("replica"), TransactionId: 1, }, { Keyspace: "b", Shard: "1", TabletType: topo.TabletType("master"), TransactionId: 2, }}, } type reflectSession struct { InTransaction bool ShardSessions []*ShardSession } type extraSession struct {
func (wr *Wrangler) exportVtnsToZkns(ctx context.Context, zconn zk.Conn, vtnsAddrPath, zknsAddrPath string) ([]string, error) { zknsPaths := make([]string, 0, 32) parts := strings.Split(vtnsAddrPath, "/") if len(parts) != 8 && len(parts) != 9 { return nil, fmt.Errorf("Invalid leaf zk path: %v", vtnsAddrPath) } cell := parts[2] keyspace := parts[5] shard := parts[6] tabletType := topo.TabletType(parts[7]) if tabletType == "action" || tabletType == "actionlog" { return nil, nil } addrs, _, err := wr.ts.GetEndPoints(ctx, cell, keyspace, shard, tabletType) if err != nil { return nil, err } // Write the individual endpoints and compute the SRV entries. vtoccAddrs := LegacyZknsAddrs{make([]string, 0, 8)} defaultAddrs := LegacyZknsAddrs{make([]string, 0, 8)} for i, entry := range addrs.Entries { zknsAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, i) zknsPaths = append(zknsPaths, zknsAddrPath) zknsAddr := zkns.ZknsAddr{ Host: entry.Host, PortMap: entry.PortMap, } err := writeAddr(zconn, zknsAddrPath, &zknsAddr) if err != nil { return nil, err } defaultAddrs.Endpoints = append(defaultAddrs.Endpoints, zknsAddrPath) vtoccAddrs.Endpoints = append(vtoccAddrs.Endpoints, zknsAddrPath+":vt") } // Prune any zkns entries that are no longer referenced by the // shard graph. deleteIdx := len(addrs.Entries) for { zknsStaleAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, deleteIdx) // A "delete" is a write of sorts - just communicate up that nothing // needs to be done to this node. zknsPaths = append(zknsPaths, zknsStaleAddrPath) err := zconn.Delete(zknsStaleAddrPath, -1) if zookeeper.IsError(err, zookeeper.ZNONODE) { break } if err != nil { return nil, err } deleteIdx++ } // Write the VDNS entries for both vtocc and mysql vtoccVdnsPath := fmt.Sprintf("%v/vt.vdns", zknsAddrPath) zknsPaths = append(zknsPaths, vtoccVdnsPath) if err = writeAddrs(zconn, vtoccVdnsPath, &vtoccAddrs); err != nil { return nil, err } defaultVdnsPath := fmt.Sprintf("%v.vdns", zknsAddrPath) zknsPaths = append(zknsPaths, defaultVdnsPath) if err = writeAddrs(zconn, defaultVdnsPath, &defaultAddrs); err != nil { return nil, err } return zknsPaths, nil }
// InitTablet initializes the tablet record if necessary. func (agent *ActionAgent) InitTablet(port, securePort, gRPCPort int) error { // only enabled if one of init_tablet_type (when healthcheck // is disabled) or init_keyspace (when healthcheck is enabled) // is passed in, then check other parameters if *initTabletType == "" && *initKeyspace == "" { return nil } // figure out our default target type var tabletType topo.TabletType if *initTabletType != "" { if *targetTabletType != "" { log.Fatalf("cannot specify both target_tablet_type and init_tablet_type parameters (as they might conflict)") } // use the type specified on the command line tabletType = topo.TabletType(*initTabletType) if !topo.IsTypeInList(tabletType, topo.AllTabletTypes) { log.Fatalf("InitTablet encountered unknown init_tablet_type '%v'", *initTabletType) } if tabletType == topo.TYPE_MASTER || tabletType == topo.TYPE_SCRAP { // We disallow TYPE_MASTER, so we don't have to change // shard.MasterAlias, and deal with the corner cases. // We also disallow TYPE_SCRAP, obviously. log.Fatalf("init_tablet_type cannot be %v", tabletType) } } else if *targetTabletType != "" { if tabletType := topo.TabletType(*targetTabletType); tabletType == topo.TYPE_MASTER { log.Fatalf("target_tablet_type cannot be '%v'. Use '%v' instead.", tabletType, topo.TYPE_REPLICA) } // use spare, the healthcheck will turn us into what // we need to be eventually tabletType = topo.TYPE_SPARE } else { log.Fatalf("if init tablet is enabled, one of init_tablet_type or target_tablet_type needs to be specified") } // create a context for this whole operation ctx, cancel := context.WithTimeout(agent.batchCtx, *initTimeout) defer cancel() // if we're assigned to a shard, make sure it exists, see if // we are its master, and update its cells list if necessary if tabletType != topo.TYPE_IDLE { if *initKeyspace == "" || *initShard == "" { log.Fatalf("if init tablet is enabled and the target type is not idle, init_keyspace and init_shard also need to be specified") } shard, _, err := topo.ValidateShardName(*initShard) if err != nil { log.Fatalf("cannot validate shard name: %v", err) } log.Infof("Reading shard record %v/%v", *initKeyspace, shard) // read the shard, create it if necessary si, err := topotools.GetOrCreateShard(ctx, agent.TopoServer, *initKeyspace, shard) if err != nil { return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err) } if si.MasterAlias == agent.TabletAlias { // we are the current master for this shard (probably // means the master tablet process was just restarted), // so InitTablet as master. tabletType = topo.TYPE_MASTER } // See if we need to add the tablet's cell to the shard's cell // list. If we do, it has to be under the shard lock. if !si.HasCell(agent.TabletAlias.Cell) { actionNode := actionnode.UpdateShard() lockPath, err := actionNode.LockShard(ctx, agent.TopoServer, *initKeyspace, shard) if err != nil { return fmt.Errorf("LockShard(%v/%v) failed: %v", *initKeyspace, shard, err) } // re-read the shard with the lock si, err = agent.TopoServer.GetShard(ctx, *initKeyspace, shard) if err != nil { return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err) } // see if we really need to update it now if !si.HasCell(agent.TabletAlias.Cell) { si.Cells = append(si.Cells, agent.TabletAlias.Cell) // write it back if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil { return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err) } } // and unlock if err := actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, nil); err != nil { return err } } } log.Infof("Initializing the tablet for type %v", tabletType) // figure out the hostname hostname := *tabletHostname if hostname == "" { var err error hostname, err = netutil.FullyQualifiedHostname() if err != nil { return err } } // create and populate tablet record tablet := &topo.Tablet{ Alias: agent.TabletAlias, Hostname: hostname, Portmap: make(map[string]int), Keyspace: *initKeyspace, Shard: *initShard, Type: tabletType, DbNameOverride: *initDbNameOverride, Tags: initTags, } if port != 0 { tablet.Portmap["vt"] = port } if securePort != 0 { tablet.Portmap["vts"] = securePort } if gRPCPort != 0 { tablet.Portmap["grpc"] = gRPCPort } if err := tablet.Complete(); err != nil { return fmt.Errorf("InitTablet tablet.Complete failed: %v", err) } // now try to create the record err := topo.CreateTablet(ctx, agent.TopoServer, tablet) switch err { case nil: // it worked, we're good, can update the replication graph if tablet.IsInReplicationGraph() { if err := topo.UpdateTabletReplicationData(ctx, agent.TopoServer, tablet); err != nil { return fmt.Errorf("UpdateTabletReplicationData failed: %v", err) } } case topo.ErrNodeExists: // The node already exists, will just try to update // it. So we read it first. oldTablet, err := agent.TopoServer.GetTablet(ctx, tablet.Alias) if err != nil { fmt.Errorf("InitTablet failed to read existing tablet record: %v", err) } // Sanity check the keyspace and shard if oldTablet.Keyspace != tablet.Keyspace || oldTablet.Shard != tablet.Shard { return fmt.Errorf("InitTablet failed because existing tablet keyspace and shard %v/%v differ from the provided ones %v/%v", oldTablet.Keyspace, oldTablet.Shard, tablet.Keyspace, tablet.Shard) } // And overwrite the rest *(oldTablet.Tablet) = *tablet if err := topo.UpdateTablet(ctx, agent.TopoServer, oldTablet); err != nil { return fmt.Errorf("UpdateTablet failed: %v", err) } // Note we don't need to UpdateTabletReplicationData // as the tablet already existed with the right data // in the replication graph default: return fmt.Errorf("CreateTablet failed: %v", err) } // and now update the serving graph. Note we do that in any case, // to clean any inaccurate record from any part of the serving graph. if tabletType != topo.TYPE_IDLE { if err := topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, tablet); err != nil { return fmt.Errorf("UpdateTabletEndpoints failed: %v", err) } } return nil }
func main() { flag.Parse() servenv.Init() defer servenv.Close() templateLoader = NewTemplateLoader(*templateDir, dummyTemplate, *debug) ts = topo.GetServer() defer topo.CloseServers() wr := wrangler.New(logutil.NewConsoleLogger(), ts, 30*time.Second, 30*time.Second) actionRepo = NewActionRepository(ts) // keyspace actions actionRepo.RegisterKeyspaceAction("ValidateKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateKeyspace(keyspace, false) }) actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateSchemaKeyspace(keyspace, nil, false) }) actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateVersionKeyspace(keyspace) }) actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidatePermissionsKeyspace(keyspace) }) // shard actions actionRepo.RegisterShardAction("ValidateShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateShard(keyspace, shard, false) }) actionRepo.RegisterShardAction("ValidateSchemaShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateSchemaShard(keyspace, shard, nil, false) }) actionRepo.RegisterShardAction("ValidateVersionShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateVersionShard(keyspace, shard) }) actionRepo.RegisterShardAction("ValidatePermissionsShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidatePermissionsShard(keyspace, shard) }) // tablet actions actionRepo.RegisterTabletAction("RpcPing", "", func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { return "", wr.ActionInitiator().RpcPing(tabletAlias, 10*time.Second) }) actionRepo.RegisterTabletAction("ScrapTablet", acl.ADMIN, func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { // refuse to scrap tablets that are not spare ti, err := wr.TopoServer().GetTablet(tabletAlias) if err != nil { return "", err } if ti.Type != topo.TYPE_SPARE { return "", fmt.Errorf("Can only scrap spare tablets") } actionPath, err := wr.Scrap(tabletAlias, false, false) if err != nil { return "", err } return "", wr.WaitForCompletion(actionPath) }) actionRepo.RegisterTabletAction("ScrapTabletForce", acl.ADMIN, func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { // refuse to scrap tablets that are not spare ti, err := wr.TopoServer().GetTablet(tabletAlias) if err != nil { return "", err } if ti.Type != topo.TYPE_SPARE { return "", fmt.Errorf("Can only scrap spare tablets") } _, err = wr.Scrap(tabletAlias, true, false) return "", err }) actionRepo.RegisterTabletAction("DeleteTablet", acl.ADMIN, func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { return "", wr.DeleteTablet(tabletAlias) }) // toplevel index http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { templateLoader.ServeTemplate("index.html", indexContent, w, r) }) // keyspace actions http.HandleFunc("/keyspace_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "no keyspace provided", http.StatusBadRequest) return } result := actionRepo.ApplyKeyspaceAction(action, keyspace, r) templateLoader.ServeTemplate("action.html", result, w, r) }) // shard actions http.HandleFunc("/shard_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "no keyspace provided", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "no shard provided", http.StatusBadRequest) return } result := actionRepo.ApplyShardAction(action, keyspace, shard, r) templateLoader.ServeTemplate("action.html", result, w, r) }) // tablet actions http.HandleFunc("/tablet_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } alias := r.FormValue("alias") if alias == "" { http.Error(w, "no alias provided", http.StatusBadRequest) return } tabletAlias, err := topo.ParseTabletAliasString(alias) if err != nil { http.Error(w, "bad alias provided", http.StatusBadRequest) return } result := actionRepo.ApplyTabletAction(action, tabletAlias, r) templateLoader.ServeTemplate("action.html", result, w, r) }) // topology server http.HandleFunc("/dbtopo", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } result := DbTopologyResult{} topology, err := topotools.DbTopology(wr.TopoServer()) if err != nil { result.Error = err.Error() } else { result.Topology = topology } templateLoader.ServeTemplate("dbtopo.html", result, w, r) }) // serving graph http.HandleFunc("/serving_graph/", func(w http.ResponseWriter, r *http.Request) { parts := strings.Split(r.URL.Path, "/") cell := parts[len(parts)-1] if cell == "" { cells, err := ts.GetKnownCells() if err != nil { httpError(w, "cannot get known cells: %v", err) return } else { templateLoader.ServeTemplate("serving_graph_cells.html", cells, w, r) } return } servingGraph := topotools.DbServingGraph(wr.TopoServer(), cell) templateLoader.ServeTemplate("serving_graph.html", servingGraph, w, r) }) // redirects for explorers http.HandleFunc("/explorers/redirect", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } var explorer Explorer switch len(explorers) { case 0: http.Error(w, "no explorer configured", http.StatusInternalServerError) return case 1: for _, ex := range explorers { explorer = ex } default: explorerName := r.FormValue("explorer") var ok bool explorer, ok = explorers[explorerName] if !ok { http.Error(w, "bad explorer name", http.StatusBadRequest) return } } var target string switch r.FormValue("type") { case "keyspace": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetKeyspacePath(keyspace) case "shard": keyspace, shard := r.FormValue("keyspace"), r.FormValue("shard") if keyspace == "" || shard == "" { http.Error(w, "keyspace and shard are obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetShardPath(keyspace, shard) case "srv_keyspace": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvKeyspacePath(cell, keyspace) case "srv_shard": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvShardPath(cell, keyspace, shard) case "srv_type": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } tabletType := r.FormValue("tablet_type") if tabletType == "" { http.Error(w, "tablet_type is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType)) case "tablet": aliasName := r.FormValue("alias") if aliasName == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } alias, err := topo.ParseTabletAliasString(aliasName) if err != nil { http.Error(w, "bad tablet alias", http.StatusBadRequest) return } target = explorer.GetTabletPath(alias) case "replication": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetReplicationSlaves(cell, keyspace, shard) default: http.Error(w, "bad redirect type", http.StatusBadRequest) return } http.Redirect(w, r, target, http.StatusFound) }) servenv.RunDefault() }
func main() { flag.Parse() servenv.Init() defer servenv.Close() templateLoader = NewTemplateLoader(*templateDir, dummyTemplate, *debug) ts := topo.GetServer() defer topo.CloseServers() wr := wrangler.New(ts, 30*time.Second, 30*time.Second) actionRepo = NewActionRepository(wr) // keyspace actions actionRepo.RegisterKeyspaceAction("ValidateKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateKeyspace(keyspace, false) }) actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateSchemaKeyspace(keyspace, false) }) actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateVersionKeyspace(keyspace) }) actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidatePermissionsKeyspace(keyspace) }) // shard actions actionRepo.RegisterShardAction("ValidateShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateShard(keyspace, shard, false) }) actionRepo.RegisterShardAction("ValidateSchemaShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateSchemaShard(keyspace, shard, false) }) actionRepo.RegisterShardAction("ValidateVersionShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateVersionShard(keyspace, shard) }) actionRepo.RegisterShardAction("ValidatePermissionsShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidatePermissionsShard(keyspace, shard) }) // tablet actions actionRepo.RegisterTabletAction("RpcPing", func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { return "", wr.ActionInitiator().RpcPing(tabletAlias, 10*time.Second) }) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { templateLoader.ServeTemplate("index.html", indexContent, w, r) }) http.HandleFunc("/keyspace_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "no keyspace provided", http.StatusBadRequest) return } result := actionRepo.ApplyKeyspaceAction(action, keyspace, r) templateLoader.ServeTemplate("action.html", result, w, r) }) http.HandleFunc("/shard_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "no keyspace provided", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "no shard provided", http.StatusBadRequest) return } result := actionRepo.ApplyShardAction(action, keyspace, shard, r) templateLoader.ServeTemplate("action.html", result, w, r) }) http.HandleFunc("/tablet_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } alias := r.FormValue("alias") if alias == "" { http.Error(w, "no alias provided", http.StatusBadRequest) return } tabletAlias, err := topo.ParseTabletAliasString(alias) if err != nil { http.Error(w, "bad alias provided", http.StatusBadRequest) return } result := actionRepo.ApplyTabletAction(action, tabletAlias, r) templateLoader.ServeTemplate("action.html", result, w, r) }) http.HandleFunc("/dbtopo", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } result := DbTopologyResult{} topology, err := wr.DbTopology() if err != nil { result.Error = err.Error() } else { result.Topology = topology } templateLoader.ServeTemplate("dbtopo.html", result, w, r) }) http.HandleFunc("/serving_graph/", func(w http.ResponseWriter, r *http.Request) { parts := strings.Split(r.URL.Path, "/") cell := parts[len(parts)-1] if cell == "" { cells, err := wr.TopoServer().GetKnownCells() if err != nil { httpError(w, "cannot get known cells: %v", err) return } else { templateLoader.ServeTemplate("serving_graph_cells.html", cells, w, r) } return } result := ServingGraphResult{} servingGraph, err := wr.ServingGraph(cell) if err != nil { result.Error = err.Error() } else { result.ServingGraph = servingGraph } templateLoader.ServeTemplate("serving_graph.html", result, w, r) }) http.HandleFunc("/explorers/redirect", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } explorerName := r.FormValue("explorer") explorer, ok := explorers[explorerName] if !ok { http.Error(w, "bad explorer name", http.StatusBadRequest) return } var target string switch r.FormValue("type") { case "keyspace": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetKeyspacePath(keyspace) case "shard": keyspace, shard := r.FormValue("keyspace"), r.FormValue("shard") if keyspace == "" || shard == "" { http.Error(w, "keyspace and shard are obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetShardPath(keyspace, shard) case "srv_keyspace": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvKeyspacePath(cell, keyspace) case "srv_shard": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvShardPath(cell, keyspace, shard) case "srv_type": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } tabletType := r.FormValue("tablet_type") if tabletType == "" { http.Error(w, "tablet_type is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType)) case "tablet": aliasName := r.FormValue("alias") if aliasName == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } alias, err := topo.ParseTabletAliasString(aliasName) if err != nil { http.Error(w, "bad tablet alias", http.StatusBadRequest) return } target = explorer.GetTabletPath(alias) case "replication": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetReplicationSlaves(cell, keyspace, shard) default: http.Error(w, "bad redirect type", http.StatusBadRequest) return } http.Redirect(w, r, target, http.StatusFound) }) servenv.Run(*port) }
// This function should only be used with an action lock on the keyspace // - otherwise the consistency of the serving graph data can't be // guaranteed. // // Take data from the global keyspace and rebuild the local serving // copies in each cell. func (wr *Wrangler) rebuildKeyspace(keyspace string, cells []string) error { relog.Info("rebuildKeyspace %v", keyspace) shards, err := wr.ts.GetShardNames(keyspace) if err != nil { return err } // Rebuild all shards in parallel. wg := sync.WaitGroup{} er := concurrency.FirstErrorRecorder{} for _, shard := range shards { wg.Add(1) go func(shard string) { if err := wr.RebuildShardGraph(keyspace, shard, cells); err != nil { er.RecordError(fmt.Errorf("RebuildShardGraph failed: %v/%v %v", keyspace, shard, err)) } wg.Done() }(shard) } wg.Wait() if er.HasErrors() { return er.Error() } // Scan the first shard to discover which cells need local serving data. aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, keyspace, shards[0]) if err != nil { return err } // srvKeyspaceByPath is a map: // key: local keyspace {cell,keyspace} // value: topo.SrvKeyspace object being built srvKeyspaceByPath := make(map[cellKeyspace]*topo.SrvKeyspace) for _, alias := range aliases { keyspaceLocation := cellKeyspace{alias.Cell, keyspace} if _, ok := srvKeyspaceByPath[keyspaceLocation]; !ok { // before adding keyspaceLocation to the map of // of KeyspaceByPath, we check this is a // serving tablet. No serving tablet in shard // 0 means we're not rebuilding the serving // graph in that cell. This is somewhat // expensive, but we only do it on all the // non-serving tablets in a shard before we // find a serving tablet. ti, err := wr.ts.GetTablet(alias) if err != nil { return err } if !ti.IsServingType() { continue } srvKeyspaceByPath[keyspaceLocation] = &topo.SrvKeyspace{Shards: make([]topo.SrvShard, 0, 16)} } } // for each entry in the srvKeyspaceByPath map, we do the following: // - read the ShardInfo structures for each shard // - prune the AddrsByType field, result would be too big // - compute the union of the db types (replica, master, ...) // - sort the shards in the list by range // - check the ranges are compatible (no hole, covers everything) for srvPath, srvKeyspace := range srvKeyspaceByPath { keyspaceDbTypes := make(map[topo.TabletType]bool) for _, shard := range shards { srvShard, err := wr.ts.GetSrvShard(srvPath.cell, srvPath.keyspace, shard) if err != nil { return err } for dbType, _ := range srvShard.AddrsByType { keyspaceDbTypes[topo.TabletType(dbType)] = true } // Prune addrs, this is unnecessarily expensive right now. It is easier to // load on-demand since we have to do that anyway on a reconnect. srvShard.AddrsByType = nil srvKeyspace.Shards = append(srvKeyspace.Shards, *srvShard) } tabletTypes := make([]topo.TabletType, 0, len(keyspaceDbTypes)) for dbType, _ := range keyspaceDbTypes { tabletTypes = append(tabletTypes, dbType) } srvKeyspace.TabletTypes = tabletTypes // FIXME(msolomon) currently this only works when the shards are range-based topo.SrvShardArray(srvKeyspace.Shards).Sort() // check the first Start is MinKey, the last End is MaxKey, // and the values in between match: End[i] == Start[i+1] if srvKeyspace.Shards[0].KeyRange.Start != key.MinKey { return fmt.Errorf("Keyspace does not start with %v", key.MinKey) } if srvKeyspace.Shards[len(srvKeyspace.Shards)-1].KeyRange.End != key.MaxKey { return fmt.Errorf("Keyspace does not end with %v", key.MaxKey) } for i, _ := range srvKeyspace.Shards[0 : len(srvKeyspace.Shards)-1] { if srvKeyspace.Shards[i].KeyRange.End != srvKeyspace.Shards[i+1].KeyRange.Start { return fmt.Errorf("Non-contiguous KeyRange values at shard %v to %v: %v != %v", i, i+1, srvKeyspace.Shards[i].KeyRange.End.Hex(), srvKeyspace.Shards[i+1].KeyRange.Start.Hex()) } } } // and then finally save the keyspace objects for srvPath, srvKeyspace := range srvKeyspaceByPath { if err := wr.ts.UpdateSrvKeyspace(srvPath.cell, srvPath.keyspace, srvKeyspace); err != nil { return fmt.Errorf("writing serving data failed: %v", err) } } return nil }