func TestQueryShard(t *testing.T) {
reflected, err := bson.Marshal(&reflectQueryShard{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
Shards: []string{"shard1", "shard2"},
TabletType: topo.TabletType("replica"),
Session: &commonSession,
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := QueryShard{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
Keyspace: "keyspace",
Shards: []string{"shard1", "shard2"},
TabletType: topo.TabletType("replica"),
Session: &commonSession,
}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
}
var unmarshalled QueryShard
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled)
}
extra, err := bson.Marshal(&extraQueryShard{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(extra, &unmarshalled)
if err != nil {
t.Error(err)
}
}
func TestSession(t *testing.T) {
reflected, err := bson.Marshal(&reflectSession{
InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := commonSession
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
}
var unmarshalled Session
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled)
}
extra, err := bson.Marshal(&extraSession{})
if err != nil {
t.Fatal(err)
}
err = bson.Unmarshal(extra, &unmarshalled)
if err != nil {
t.Error(err)
}
}
func getSrvKeyspace(rpcClient *rpcplus.Client, cell, keyspace string, verbose bool) {
req := &topo.GetSrvKeyspaceArgs{
Cell: cell,
Keyspace: keyspace,
}
reply := &topo.SrvKeyspace{}
if err := rpcClient.Call(context.TODO(), "TopoReader.GetSrvKeyspace", req, reply); err != nil {
log.Fatalf("TopoReader.GetSrvKeyspace error: %v", err)
}
if verbose {
tabletTypes := make([]string, 0, len(reply.Partitions))
for t, _ := range reply.Partitions {
tabletTypes = append(tabletTypes, string(t))
}
sort.Strings(tabletTypes)
for _, t := range tabletTypes {
println(fmt.Sprintf("Partitions[%v] =", t))
for i, s := range reply.Partitions[topo.TabletType(t)].Shards {
println(fmt.Sprintf(" Shards[%v]=%v", i, s.KeyRange.String()))
}
}
for i, s := range reply.Shards {
println(fmt.Sprintf("Shards[%v]=%v", i, s.KeyRange.String()))
}
for i, t := range reply.TabletTypes {
println(fmt.Sprintf("TabletTypes[%v] = %v", i, t))
}
}
}
func (agent *ActionAgent) initHeathCheck() {
if !agent.IsRunningHealthCheck() {
log.Infof("No target_tablet_type specified, disabling any health check")
return
}
log.Infof("Starting periodic health check every %v with target_tablet_type=%v", *healthCheckInterval, *targetTabletType)
t := timer.NewTimer(*healthCheckInterval)
servenv.OnTermSync(func() {
// When we enter lameduck mode, we want to not call
// the health check any more. After this returns, we
// are guaranteed to not call it.
log.Info("Stopping periodic health check timer")
t.Stop()
// Now we can finish up and force ourselves to not healthy.
agent.terminateHealthChecks(topo.TabletType(*targetTabletType))
})
t.Start(func() {
agent.runHealthCheck(topo.TabletType(*targetTabletType))
})
}
func getEndPoints(rpcClient *rpcplus.Client, cell, keyspace, shard, tabletType string, verbose bool) {
req := &topo.GetEndPointsArgs{
Cell: cell,
Keyspace: keyspace,
Shard: shard,
TabletType: topo.TabletType(tabletType),
}
reply := &topo.EndPoints{}
if err := rpcClient.Call(context.TODO(), "TopoReader.GetEndPoints", req, reply); err != nil {
log.Fatalf("TopoReader.GetEndPoints error: %v", err)
}
if verbose {
for i, e := range reply.Entries {
println(fmt.Sprintf("Entries[%v] = %v %v", i, e.Uid, e.Host))
}
}
}
// for direct zk connection: vtzk://host:port/cell/keyspace/tabletType
// we always use a MetaConn, host and port are ignored.
// the driver name dictates if we use zk or zkocc, and streaming or not
func (driver *sDriver) Open(name string) (sc db.Conn, err error) {
if !strings.HasPrefix(name, "vtzk://") {
// add a default protocol talking to zk
name = "vtzk://" + name
}
u, err := url.Parse(name)
if err != nil {
return nil, err
}
dbi, tabletType := path.Split(u.Path)
dbi = strings.Trim(dbi, "/")
tabletType = strings.Trim(tabletType, "/")
cell, keyspace := path.Split(dbi)
cell = strings.Trim(cell, "/")
keyspace = strings.Trim(keyspace, "/")
return Dial(driver.ts, cell, keyspace, topo.TabletType(tabletType), driver.stream, tablet.DefaultTimeout)
}
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) {
zkSgShardPath := zkPathForVtShard(cell, keyspace, shard)
children, _, err := zkts.zconn.Children(zkSgShardPath)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
err = topo.ErrNoNode
}
return nil, err
}
result := make([]topo.TabletType, 0, len(children))
for _, tt := range children {
// these two are used for locking
if tt == "action" || tt == "actionlog" {
continue
}
result = append(result, topo.TabletType(tt))
}
return result, nil
}
func TestKeyspaceIdBatchQuery(t *testing.T) {
reflected, err := bson.Marshal(&reflectKeyspaceIdBatchQuery{
Queries: []reflectBoundQuery{{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
}},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")},
Session: &Session{InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
},
})
if err != nil {
t.Error(err)
}
want := string(reflected)
custom := KeyspaceIdBatchQuery{
Queries: []tproto.BoundQuery{{
Sql: "query",
BindVariables: map[string]interface{}{"val": int64(1)},
}},
Keyspace: "keyspace",
KeyspaceIds: []kproto.KeyspaceId{kproto.KeyspaceId("10"), kproto.KeyspaceId("20")},
Session: &commonSession,
}
encoded, err := bson.Marshal(&custom)
if err != nil {
t.Error(err)
}
got := string(encoded)
if want != got {
t.Errorf("want\n%+v, got\n%+v", want, got)
}
var unmarshalled KeyspaceIdBatchQuery
err = bson.Unmarshal(encoded, &unmarshalled)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(custom, unmarshalled) {
t.Errorf("want \n%+v, got \n%+v", custom, unmarshalled)
}
extra, err := bson.Marshal(&extraKeyspaceIdBatchQuery{})
if err != nil {
t.Error(err)
}
err = bson.Unmarshal(extra, &unmarshalled)
if err != nil {
t.Error(err)
}
}
"testing"
"github.com/henryanand/vitess/go/bson"
mproto "github.com/henryanand/vitess/go/mysql/proto"
"github.com/henryanand/vitess/go/sqltypes"
kproto "github.com/henryanand/vitess/go/vt/key"
tproto "github.com/henryanand/vitess/go/vt/tabletserver/proto"
"github.com/henryanand/vitess/go/vt/topo"
)
var commonSession = Session{
InTransaction: true,
ShardSessions: []*ShardSession{{
Keyspace: "a",
Shard: "0",
TabletType: topo.TabletType("replica"),
TransactionId: 1,
}, {
Keyspace: "b",
Shard: "1",
TabletType: topo.TabletType("master"),
TransactionId: 2,
}},
}
type reflectSession struct {
InTransaction bool
ShardSessions []*ShardSession
}
type extraSession struct {
func (wr *Wrangler) exportVtnsToZkns(zconn zk.Conn, vtnsAddrPath, zknsAddrPath string) ([]string, error) {
zknsPaths := make([]string, 0, 32)
parts := strings.Split(vtnsAddrPath, "/")
if len(parts) != 8 && len(parts) != 9 {
return nil, fmt.Errorf("Invalid leaf zk path: %v", vtnsAddrPath)
}
cell := parts[2]
keyspace := parts[5]
shard := parts[6]
tabletType := topo.TabletType(parts[7])
if tabletType == "action" || tabletType == "actionlog" {
return nil, nil
}
addrs, err := wr.ts.GetEndPoints(cell, keyspace, shard, tabletType)
if err != nil {
return nil, err
}
// Write the individual endpoints and compute the SRV entries.
vtoccAddrs := LegacyZknsAddrs{make([]string, 0, 8)}
defaultAddrs := LegacyZknsAddrs{make([]string, 0, 8)}
for i, entry := range addrs.Entries {
zknsAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, i)
zknsPaths = append(zknsPaths, zknsAddrPath)
zknsAddr := zkns.ZknsAddr{Host: entry.Host, Port: entry.NamedPortMap["_mysql"], NamedPortMap: entry.NamedPortMap}
err := WriteAddr(zconn, zknsAddrPath, &zknsAddr)
if err != nil {
return nil, err
}
defaultAddrs.Endpoints = append(defaultAddrs.Endpoints, zknsAddrPath)
vtoccAddrs.Endpoints = append(vtoccAddrs.Endpoints, zknsAddrPath+":_vtocc")
}
// Prune any zkns entries that are no longer referenced by the
// shard graph.
deleteIdx := len(addrs.Entries)
for {
zknsStaleAddrPath := fmt.Sprintf("%v/%v", zknsAddrPath, deleteIdx)
// A "delete" is a write of sorts - just communicate up that nothing
// needs to be done to this node.
zknsPaths = append(zknsPaths, zknsStaleAddrPath)
err := zconn.Delete(zknsStaleAddrPath, -1)
if zookeeper.IsError(err, zookeeper.ZNONODE) {
break
}
if err != nil {
return nil, err
}
deleteIdx++
}
// Write the VDNS entries for both vtocc and mysql
vtoccVdnsPath := fmt.Sprintf("%v/_vtocc.vdns", zknsAddrPath)
zknsPaths = append(zknsPaths, vtoccVdnsPath)
if err = WriteAddrs(zconn, vtoccVdnsPath, &vtoccAddrs); err != nil {
return nil, err
}
defaultVdnsPath := fmt.Sprintf("%v.vdns", zknsAddrPath)
zknsPaths = append(zknsPaths, defaultVdnsPath)
if err = WriteAddrs(zconn, defaultVdnsPath, &defaultAddrs); err != nil {
return nil, err
}
return zknsPaths, nil
}
func main() {
flag.Parse()
servenv.Init()
defer servenv.Close()
templateLoader = NewTemplateLoader(*templateDir, dummyTemplate, *debug)
ts = topo.GetServer()
defer topo.CloseServers()
wr := wrangler.New(logutil.NewConsoleLogger(), ts, 30*time.Second, 30*time.Second)
actionRepo = NewActionRepository(ts)
// keyspace actions
actionRepo.RegisterKeyspaceAction("ValidateKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateKeyspace(keyspace, false)
})
actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateSchemaKeyspace(keyspace, nil, false)
})
actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidateVersionKeyspace(keyspace)
})
actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace",
func(wr *wrangler.Wrangler, keyspace string, r *http.Request) (string, error) {
return "", wr.ValidatePermissionsKeyspace(keyspace)
})
// shard actions
actionRepo.RegisterShardAction("ValidateShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateShard(keyspace, shard, false)
})
actionRepo.RegisterShardAction("ValidateSchemaShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateSchemaShard(keyspace, shard, nil, false)
})
actionRepo.RegisterShardAction("ValidateVersionShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidateVersionShard(keyspace, shard)
})
actionRepo.RegisterShardAction("ValidatePermissionsShard",
func(wr *wrangler.Wrangler, keyspace, shard string, r *http.Request) (string, error) {
return "", wr.ValidatePermissionsShard(keyspace, shard)
})
// tablet actions
actionRepo.RegisterTabletAction("Ping", "",
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
ti, err := wr.TopoServer().GetTablet(tabletAlias)
if err != nil {
return "", err
}
return "", wr.TabletManagerClient().Ping(wr.Context(), ti)
})
actionRepo.RegisterTabletAction("ScrapTablet", acl.ADMIN,
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
// refuse to scrap tablets that are not spare
ti, err := wr.TopoServer().GetTablet(tabletAlias)
if err != nil {
return "", err
}
if ti.Type != topo.TYPE_SPARE {
return "", fmt.Errorf("Can only scrap spare tablets")
}
return "", wr.Scrap(tabletAlias, false, false)
})
actionRepo.RegisterTabletAction("ScrapTabletForce", acl.ADMIN,
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
// refuse to scrap tablets that are not spare
ti, err := wr.TopoServer().GetTablet(tabletAlias)
if err != nil {
return "", err
}
if ti.Type != topo.TYPE_SPARE {
return "", fmt.Errorf("Can only scrap spare tablets")
}
return "", wr.Scrap(tabletAlias, true, false)
})
actionRepo.RegisterTabletAction("DeleteTablet", acl.ADMIN,
func(wr *wrangler.Wrangler, tabletAlias topo.TabletAlias, r *http.Request) (string, error) {
return "", wr.DeleteTablet(tabletAlias)
})
// toplevel index
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
templateLoader.ServeTemplate("index.html", indexContent, w, r)
})
// keyspace actions
http.HandleFunc("/keyspace_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "no keyspace provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyKeyspaceAction(action, keyspace, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
// shard actions
http.HandleFunc("/shard_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "no keyspace provided", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "no shard provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyShardAction(action, keyspace, shard, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
// tablet actions
http.HandleFunc("/tablet_actions", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
action := r.FormValue("action")
if action == "" {
http.Error(w, "no action provided", http.StatusBadRequest)
return
}
alias := r.FormValue("alias")
if alias == "" {
http.Error(w, "no alias provided", http.StatusBadRequest)
return
}
tabletAlias, err := topo.ParseTabletAliasString(alias)
if err != nil {
http.Error(w, "bad alias provided", http.StatusBadRequest)
return
}
result := actionRepo.ApplyTabletAction(action, tabletAlias, r)
templateLoader.ServeTemplate("action.html", result, w, r)
})
// topology server
http.HandleFunc("/dbtopo", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
result := DbTopologyResult{}
topology, err := topotools.DbTopology(context.TODO(), wr.TopoServer())
if err != nil {
result.Error = err.Error()
} else {
result.Topology = topology
}
templateLoader.ServeTemplate("dbtopo.html", result, w, r)
})
// serving graph
http.HandleFunc("/serving_graph/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(r.URL.Path, "/")
cell := parts[len(parts)-1]
if cell == "" {
cells, err := ts.GetKnownCells()
if err != nil {
httpError(w, "cannot get known cells: %v", err)
return
} else {
templateLoader.ServeTemplate("serving_graph_cells.html", cells, w, r)
}
return
}
servingGraph := topotools.DbServingGraph(wr.TopoServer(), cell)
templateLoader.ServeTemplate("serving_graph.html", servingGraph, w, r)
})
// redirects for explorers
http.HandleFunc("/explorers/redirect", func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
httpError(w, "cannot parse form: %s", err)
return
}
var explorer Explorer
switch len(explorers) {
case 0:
http.Error(w, "no explorer configured", http.StatusInternalServerError)
return
case 1:
for _, ex := range explorers {
explorer = ex
}
default:
explorerName := r.FormValue("explorer")
var ok bool
explorer, ok = explorers[explorerName]
if !ok {
http.Error(w, "bad explorer name", http.StatusBadRequest)
return
}
}
var target string
switch r.FormValue("type") {
case "keyspace":
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetKeyspacePath(keyspace)
case "shard":
keyspace, shard := r.FormValue("keyspace"), r.FormValue("shard")
if keyspace == "" || shard == "" {
http.Error(w, "keyspace and shard are obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetShardPath(keyspace, shard)
case "srv_keyspace":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvKeyspacePath(cell, keyspace)
case "srv_shard":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvShardPath(cell, keyspace, shard)
case "srv_type":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
tabletType := r.FormValue("tablet_type")
if tabletType == "" {
http.Error(w, "tablet_type is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetSrvTypePath(cell, keyspace, shard, topo.TabletType(tabletType))
case "tablet":
aliasName := r.FormValue("alias")
if aliasName == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
alias, err := topo.ParseTabletAliasString(aliasName)
if err != nil {
http.Error(w, "bad tablet alias", http.StatusBadRequest)
return
}
target = explorer.GetTabletPath(alias)
case "replication":
cell := r.FormValue("cell")
if cell == "" {
http.Error(w, "cell is obligatory for this redirect", http.StatusBadRequest)
return
}
keyspace := r.FormValue("keyspace")
if keyspace == "" {
http.Error(w, "keyspace is obligatory for this redirect", http.StatusBadRequest)
return
}
shard := r.FormValue("shard")
if shard == "" {
http.Error(w, "shard is obligatory for this redirect", http.StatusBadRequest)
return
}
target = explorer.GetReplicationSlaves(cell, keyspace, shard)
default:
http.Error(w, "bad redirect type", http.StatusBadRequest)
return
}
http.Redirect(w, r, target, http.StatusFound)
})
servenv.RunDefault()
}