func TestQueryShard(t *testing.T) { reflected, err := bson.Marshal(&reflectQueryShard{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, Keyspace: "keyspace", Shards: []string{"shard1", "shard2"}, TabletType: topo.TabletType("replica"), Session: &commonSession, }) if err != nil { t.Error(err) } want := string(reflected) custom := QueryShard{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, Keyspace: "keyspace", Shards: []string{"shard1", "shard2"}, TabletType: topo.TabletType("replica"), Session: &commonSession, } encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%+v, got\n%+v", want, got) } var unmarshalled QueryShard err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if !reflect.DeepEqual(custom, unmarshalled) { t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled) } extra, err := bson.Marshal(&extraQueryShard{}) if err != nil { t.Error(err) } err = bson.Unmarshal(extra, &unmarshalled) if err != nil { t.Error(err) } }
func TestSession(t *testing.T) { reflected, err := bson.Marshal(&reflectSession{ InTransaction: true, ShardSessions: []*ShardSession{{ Keyspace: "a", Shard: "0", TabletType: topo.TabletType("replica"), TransactionId: 1, }, { Keyspace: "b", Shard: "1", TabletType: topo.TabletType("master"), TransactionId: 2, }}, }) if err != nil { t.Error(err) } want := string(reflected) custom := commonSession encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%+v, got\n%+v", want, got) } var unmarshalled Session err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if !reflect.DeepEqual(custom, unmarshalled) { t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled) } extra, err := bson.Marshal(&extraSession{}) if err != nil { t.Fatal(err) } err = bson.Unmarshal(extra, &unmarshalled) if err != nil { t.Error(err) } }
func getSrvKeyspace(rpcClient *rpcplus.Client, cell, keyspace string, verbose bool) { req := &topo.GetSrvKeyspaceArgs{ Cell: cell, Keyspace: keyspace, } reply := &topo.SrvKeyspace{} if err := rpcClient.Call(context.TODO(), "TopoReader.GetSrvKeyspace", req, reply); err != nil { log.Fatalf("TopoReader.GetSrvKeyspace error: %v", err) } if verbose { tabletTypes := make([]string, 0, len(reply.Partitions)) for t, _ := range reply.Partitions { tabletTypes = append(tabletTypes, string(t)) } sort.Strings(tabletTypes) for _, t := range tabletTypes { println(fmt.Sprintf("Partitions[%v] =", t)) for i, s := range reply.Partitions[topo.TabletType(t)].Shards { println(fmt.Sprintf(" Shards[%v]=%v", i, s.KeyRange.String())) } } for i, s := range reply.Shards { println(fmt.Sprintf("Shards[%v]=%v", i, s.KeyRange.String())) } for i, t := range reply.TabletTypes { println(fmt.Sprintf("TabletTypes[%v] = %v", i, t)) } } }
func (agent *ActionAgent) initHeathCheck() { if !agent.IsRunningHealthCheck() { log.Infof("No target_tablet_type specified, disabling any health check") return } log.Infof("Starting periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType) t := timer.NewTimer(*healthCheckInterval) servenv.OnTermSync(func() { // When we enter lameduck mode, we want to not call // the health check any more. After this returns, we // are guaranteed to not call it. log.Info("Stopping periodic health check timer") t.Stop() // Now we can finish up and force ourselves to not healthy. agent.terminateHealthChecks(topo.TabletType(*targetTabletType)) }) t.Start(func() { agent.runHealthCheck(topo.TabletType(*targetTabletType)) }) }
func getEndPoints(rpcClient *rpcplus.Client, cell, keyspace, shard, tabletType string, verbose bool) { req := &topo.GetEndPointsArgs{ Cell: cell, Keyspace: keyspace, Shard: shard, TabletType: topo.TabletType(tabletType), } reply := &topo.EndPoints{} if err := rpcClient.Call(context.TODO(), "TopoReader.GetEndPoints", req, reply); err != nil { log.Fatalf("TopoReader.GetEndPoints error: %v", err) } if verbose { for i, e := range reply.Entries { println(fmt.Sprintf("Entries[%v] = %v %v", i, e.Uid, e.Host)) } } }
// for direct zk connection: vtzk://host:port/cell/keyspace/tabletType // we always use a MetaConn, host and port are ignored. // the driver name dictates if we use zk or zkocc, and streaming or not func (driver *sDriver) Open(name string) (sc db.Conn, err error) { if !strings.HasPrefix(name, "vtzk://") { // add a default protocol talking to zk name = "vtzk://" + name } u, err := url.Parse(name) if err != nil { return nil, err } dbi, tabletType := path.Split(u.Path) dbi = strings.Trim(dbi, "/") tabletType = strings.Trim(tabletType, "/") cell, keyspace := path.Split(dbi) cell = strings.Trim(cell, "/") keyspace = strings.Trim(keyspace, "/") return Dial(driver.ts, cell, keyspace, topo.TabletType(tabletType), driver.stream, tablet.DefaultTimeout) }
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) { zkSgShardPath := zkPathForVtShard(cell, keyspace, shard) children, _, err := zkts.zconn.Children(zkSgShardPath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } return nil, err } result := make([]topo.TabletType, 0, len(children)) for _, tt := range children { // these two are used for locking if tt == "action" || tt == "actionlog" { continue } result = append(result, topo.TabletType(tt)) } return result, nil }
func TestKeyspaceIdBatchQuery(t *testing.T) { reflected, err := bson.Marshal(&reflectKeyspaceIdBatchQuery{ Queries: []reflectBoundQuery{{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, }}, Keyspace: "keyspace", KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")}, Session: &Session{InTransaction: true, ShardSessions: []*ShardSession{{ Keyspace: "a", Shard: "0", TabletType: topo.TabletType("replica"), TransactionId: 1, }, { Keyspace: "b", Shard: "1", TabletType: topo.TabletType("master"), TransactionId: 2, }}, }, }) if err != nil { t.Error(err) } want := string(reflected) custom := KeyspaceIdBatchQuery{ Queries: []tproto.BoundQuery{{ Sql: "query", BindVariables: map[string]interface{}{"val": int64(1)}, }}, Keyspace: "keyspace", KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")}, Session: &commonSession, } encoded, err := bson.Marshal(&custom) if err != nil { t.Error(err) } got := string(encoded) if want != got { t.Errorf("want\n%+v, got\n%+v", want, got) } var unmarshalled KeyspaceIdBatchQuery err = bson.Unmarshal(encoded, &unmarshalled) if err != nil { t.Error(err) } if !reflect.DeepEqual(custom, unmarshalled) { t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled) } extra, err := bson.Marshal(&extraKeyspaceIdBatchQuery{}) if err != nil { t.Error(err) } err = bson.Unmarshal(extra, &unmarshalled) if err != nil { t.Error(err) } }
"testing" "github.com/henryanand/vitess/go/bson" mproto "github.com/henryanand/vitess/go/mysql/proto" "github.com/henryanand/vitess/go/sqltypes" kproto "github.com/henryanand/vitess/go/vt/key" tproto "github.com/henryanand/vitess/go/vt/tabletserver/proto" "github.com/henryanand/vitess/go/vt/topo" ) var commonSession = Session{ InTransaction: true, ShardSessions: []*ShardSession{{ Keyspace: "a", Shard: "0", TabletType: topo.TabletType("replica"), TransactionId: 1, }, { Keyspace: "b", Shard: "1", TabletType: topo.TabletType("master"), TransactionId: 2, }}, } type reflectSession struct { InTransaction bool ShardSessions []*ShardSession } type extraSession struct {
func (wr *Wrangler) exportVtnsToZkns(zconn zk.Conn, vtnsAddrPath, zknsAddrPath string) ([]string, error) { zknsPaths := make([]string, 0, 32) parts := strings.Split(vtnsAddrPath, "/") if len(parts) != 8 && len(parts) != 9 { return nil, fmt.Errorf("Invalid leaf zk path: %v", vtnsAddrPath) } cell := parts[2] keyspace := parts[5] shard := parts[6] tabletType := topo.TabletType(parts[7]) if tabletType == "action" || tabletType == "actionlog" { return nil, nil } addrs, err := wr.ts.GetEndPoints(cell, keyspace, shard, tabletType) if err != nil { return nil, err } // Write the individual endpoints and compute the SRV entries. vtoccAddrs := LegacyZknsAddrs{make([]string, 0, 8)} defaultAddrs := LegacyZknsAddrs{make([]string, 0, 8)} for i, entry := range addrs.Entries { zknsAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, i) zknsPaths = append(zknsPaths, zknsAddrPath) zknsAddr := zkns.ZknsAddr{Host: entry.Host, Port: entry.NamedPortMap["_mysql"], NamedPortMap: entry.NamedPortMap} err := WriteAddr(zconn, zknsAddrPath, &zknsAddr) if err != nil { return nil, err } defaultAddrs.Endpoints = append(defaultAddrs.Endpoints, zknsAddrPath) vtoccAddrs.Endpoints = append(vtoccAddrs.Endpoints, zknsAddrPath+":_vtocc") } // Prune any zkns entries that are no longer referenced by the // shard graph. deleteIdx := len(addrs.Entries) for { zknsStaleAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, deleteIdx) // A "delete" is a write of sorts - just communicate up that nothing // needs to be done to this node. zknsPaths = append(zknsPaths, zknsStaleAddrPath) err := zconn.Delete(zknsStaleAddrPath, -1) if zookeeper.IsError(err, zookeeper.ZNONODE) { break } if err != nil { return nil, err } deleteIdx++ } // Write the VDNS entries for both vtocc and mysql vtoccVdnsPath := fmt.Sprintf("%v/_vtocc.vdns", zknsAddrPath) zknsPaths = append(zknsPaths, vtoccVdnsPath) if err = WriteAddrs(zconn, vtoccVdnsPath, &vtoccAddrs); err != nil { return nil, err } defaultVdnsPath := fmt.Sprintf("%v.vdns", zknsAddrPath) zknsPaths = append(zknsPaths, defaultVdnsPath) if err = WriteAddrs(zconn, defaultVdnsPath, &defaultAddrs); err != nil { return nil, err } return zknsPaths, nil }
func main() { flag.Parse() servenv.Init() defer servenv.Close() templateLoader = NewTemplateLoader(*templateDir, dummyTemplate, *debug) ts = topo.GetServer() defer topo.CloseServers() wr := wrangler.New(logutil.NewConsoleLogger(), ts, 30*time.Second, 30*time.Second) actionRepo = NewActionRepository(ts) // keyspace actions actionRepo.RegisterKeyspaceAction("ValidateKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateKeyspace(keyspace, false) }) actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateSchemaKeyspace(keyspace, nil, false) }) actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidateVersionKeyspace(keyspace) }) actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace", func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) { return "", wr.ValidatePermissionsKeyspace(keyspace) }) // shard actions actionRepo.RegisterShardAction("ValidateShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateShard(keyspace, shard, false) }) actionRepo.RegisterShardAction("ValidateSchemaShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateSchemaShard(keyspace, shard, nil, false) }) actionRepo.RegisterShardAction("ValidateVersionShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidateVersionShard(keyspace, shard) }) actionRepo.RegisterShardAction("ValidatePermissionsShard", func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) { return "", wr.ValidatePermissionsShard(keyspace, shard) }) // tablet actions actionRepo.RegisterTabletAction("Ping", "", func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { ti, err := wr.TopoServer().GetTablet(tabletAlias) if err != nil { return "", err } return "", wr.TabletManagerClient().Ping(wr.Context(), ti) }) actionRepo.RegisterTabletAction("ScrapTablet", acl.ADMIN, func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { // refuse to scrap tablets that are not spare ti, err := wr.TopoServer().GetTablet(tabletAlias) if err != nil { return "", err } if ti.Type != topo.TYPE_SPARE { return "", fmt.Errorf("Can only scrap spare tablets") } return "", wr.Scrap(tabletAlias, false, false) }) actionRepo.RegisterTabletAction("ScrapTabletForce", acl.ADMIN, func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { // refuse to scrap tablets that are not spare ti, err := wr.TopoServer().GetTablet(tabletAlias) if err != nil { return "", err } if ti.Type != topo.TYPE_SPARE { return "", fmt.Errorf("Can only scrap spare tablets") } return "", wr.Scrap(tabletAlias, true, false) }) actionRepo.RegisterTabletAction("DeleteTablet", acl.ADMIN, func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) { return "", wr.DeleteTablet(tabletAlias) }) // toplevel index http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { templateLoader.ServeTemplate("index.html", indexContent, w, r) }) // keyspace actions http.HandleFunc("/keyspace_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "no keyspace provided", http.StatusBadRequest) return } result := actionRepo.ApplyKeyspaceAction(action, keyspace, r) templateLoader.ServeTemplate("action.html", result, w, r) }) // shard actions http.HandleFunc("/shard_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "no keyspace provided", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "no shard provided", http.StatusBadRequest) return } result := actionRepo.ApplyShardAction(action, keyspace, shard, r) templateLoader.ServeTemplate("action.html", result, w, r) }) // tablet actions http.HandleFunc("/tablet_actions", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } action := r.FormValue("action") if action == "" { http.Error(w, "no action provided", http.StatusBadRequest) return } alias := r.FormValue("alias") if alias == "" { http.Error(w, "no alias provided", http.StatusBadRequest) return } tabletAlias, err := topo.ParseTabletAliasString(alias) if err != nil { http.Error(w, "bad alias provided", http.StatusBadRequest) return } result := actionRepo.ApplyTabletAction(action, tabletAlias, r) templateLoader.ServeTemplate("action.html", result, w, r) }) // topology server http.HandleFunc("/dbtopo", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } result := DbTopologyResult{} topology, err := topotools.DbTopology(context.TODO(), wr.TopoServer()) if err != nil { result.Error = err.Error() } else { result.Topology = topology } templateLoader.ServeTemplate("dbtopo.html", result, w, r) }) // serving graph http.HandleFunc("/serving_graph/", func(w http.ResponseWriter, r *http.Request) { parts := strings.Split(r.URL.Path, "/") cell := parts[len(parts)-1] if cell == "" { cells, err := ts.GetKnownCells() if err != nil { httpError(w, "cannot get known cells: %v", err) return } else { templateLoader.ServeTemplate("serving_graph_cells.html", cells, w, r) } return } servingGraph := topotools.DbServingGraph(wr.TopoServer(), cell) templateLoader.ServeTemplate("serving_graph.html", servingGraph, w, r) }) // redirects for explorers http.HandleFunc("/explorers/redirect", func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { httpError(w, "cannot parse form: %s", err) return } var explorer Explorer switch len(explorers) { case 0: http.Error(w, "no explorer configured", http.StatusInternalServerError) return case 1: for _, ex := range explorers { explorer = ex } default: explorerName := r.FormValue("explorer") var ok bool explorer, ok = explorers[explorerName] if !ok { http.Error(w, "bad explorer name", http.StatusBadRequest) return } } var target string switch r.FormValue("type") { case "keyspace": keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetKeyspacePath(keyspace) case "shard": keyspace, shard := r.FormValue("keyspace"), r.FormValue("shard") if keyspace == "" || shard == "" { http.Error(w, "keyspace and shard are obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetShardPath(keyspace, shard) case "srv_keyspace": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvKeyspacePath(cell, keyspace) case "srv_shard": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvShardPath(cell, keyspace, shard) case "srv_type": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } tabletType := r.FormValue("tablet_type") if tabletType == "" { http.Error(w, "tablet_type is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType)) case "tablet": aliasName := r.FormValue("alias") if aliasName == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } alias, err := topo.ParseTabletAliasString(aliasName) if err != nil { http.Error(w, "bad tablet alias", http.StatusBadRequest) return } target = explorer.GetTabletPath(alias) case "replication": cell := r.FormValue("cell") if cell == "" { http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest) return } keyspace := r.FormValue("keyspace") if keyspace == "" { http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest) return } shard := r.FormValue("shard") if shard == "" { http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest) return } target = explorer.GetReplicationSlaves(cell, keyspace, shard) default: http.Error(w, "bad redirect type", http.StatusBadRequest) return } http.Redirect(w, r, target, http.StatusFound) }) servenv.RunDefault() }