// FIXME(msolomon) This validate presumes the master is up and running.
// Even when that isn't true, there are validation processes that might be valuable.
func (wr *Wrangler) validateShard(ctx context.Context, keyspace, shard string, pingTablets bool, wg *sync.WaitGroup, results chan<- error) {
shardInfo, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
results <- fmt.Errorf("TopologyServer.GetShard(%v, %v) failed: %v", keyspace, shard, err)
return
}
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
results <- fmt.Errorf("TopologyServer.FindAllTabletAliasesInShard(%v, %v) failed: %v", keyspace, shard, err)
return
}
tabletMap, _ := topo.GetTabletMap(ctx, wr.ts, aliases)
var masterAlias *pb.TabletAlias
for _, alias := range aliases {
tabletInfo, ok := tabletMap[*alias]
if !ok {
results <- fmt.Errorf("tablet %v not found in map", alias)
continue
}
if tabletInfo.Type == pb.TabletType_MASTER {
if masterAlias != nil {
results <- fmt.Errorf("shard %v/%v already has master %v but found other master %v", keyspace, shard, masterAlias, alias)
} else {
masterAlias = alias
}
}
}
if masterAlias == nil {
results <- fmt.Errorf("no master for shard %v/%v", keyspace, shard)
} else if !topo.TabletAliasEqual(shardInfo.MasterAlias, masterAlias) {
results <- fmt.Errorf("master mismatch for shard %v/%v: found %v, expected %v", keyspace, shard, masterAlias, shardInfo.MasterAlias)
}
for _, alias := range aliases {
wg.Add(1)
go func(alias *pb.TabletAlias) {
defer wg.Done()
if err := topo.Validate(ctx, wr.ts, alias); err != nil {
results <- fmt.Errorf("Validate(%v) failed: %v", alias, err)
} else {
wr.Logger().Infof("tablet %v is valid", alias)
}
}(alias)
}
if pingTablets {
wr.validateReplication(ctx, shardInfo, tabletMap, results)
wr.pingTablets(ctx, tabletMap, wg, results)
}
return
}
// FIXME(msolomon) This validate presumes the master is up and running.
// Even when that isn't true, there are validation processes that might be valuable.
func (wr *Wrangler) validateShard(keyspace, shard string, pingTablets bool, wg *sync.WaitGroup, results chan<- vresult) {
shardInfo, err := wr.ts.GetShard(keyspace, shard)
if err != nil {
results <- vresult{keyspace + "/" + shard, err}
return
}
aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, keyspace, shard)
if err != nil {
results <- vresult{keyspace + "/" + shard, err}
}
tabletMap, _ := GetTabletMap(wr.ts, aliases)
var masterAlias topo.TabletAlias
for _, alias := range aliases {
tabletInfo, ok := tabletMap[alias]
if !ok {
results <- vresult{alias.String(), fmt.Errorf("tablet not found in map")}
continue
}
if tabletInfo.Parent.Uid == topo.NO_TABLET {
if masterAlias.Cell != "" {
results <- vresult{alias.String(), fmt.Errorf("tablet already has a master %v", masterAlias)}
} else {
masterAlias = alias
}
}
}
if masterAlias.Cell == "" {
results <- vresult{keyspace + "/" + shard, fmt.Errorf("no master for shard")}
} else if shardInfo.MasterAlias != masterAlias {
results <- vresult{keyspace + "/" + shard, fmt.Errorf("master mismatch for shard: found %v, expected %v", masterAlias, shardInfo.MasterAlias)}
}
for _, alias := range aliases {
tabletReplicationPath := masterAlias.String()
if alias != masterAlias {
tabletReplicationPath += "/" + alias.String()
}
wg.Add(1)
go func(alias topo.TabletAlias) {
results <- vresult{tabletReplicationPath, topo.Validate(wr.ts, alias, tabletReplicationPath)}
wg.Done()
}(alias)
}
if pingTablets {
wr.validateReplication(shardInfo, tabletMap, results)
wr.pingTablets(tabletMap, wg, results)
}
return
}
func (agent *ActionAgent) verifyTopology() error {
tablet := agent.Tablet()
if tablet == nil {
return fmt.Errorf("agent._tablet is nil")
}
if err := topo.Validate(agent.TopoServer, agent.TabletAlias); err != nil {
// Don't stop, it's not serious enough, this is likely transient.
log.Warningf("tablet validate failed: %v %v", agent.TabletAlias, err)
}
return agent.TopoServer.ValidateTabletActions(agent.TabletAlias)
}
// Validate all tablets in all discoverable cells, even if they are
// not in the replication graph.
func (wr *Wrangler) validateAllTablets(ctx context.Context, wg *sync.WaitGroup, results chan<- error) {
cellSet := make(map[string]bool, 16)
keyspaces, err := wr.ts.GetKeyspaces(ctx)
if err != nil {
results <- fmt.Errorf("TopologyServer.GetKeyspaces failed: %v", err)
return
}
for _, keyspace := range keyspaces {
shards, err := wr.ts.GetShardNames(ctx, keyspace)
if err != nil {
results <- fmt.Errorf("TopologyServer.GetShardNames(%v) failed: %v", keyspace, err)
return
}
for _, shard := range shards {
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
results <- fmt.Errorf("TopologyServer.FindAllTabletAliasesInShard(%v, %v) failed: %v", keyspace, shard, err)
return
}
for _, alias := range aliases {
cellSet[alias.Cell] = true
}
}
}
for cell := range cellSet {
aliases, err := wr.ts.GetTabletsByCell(ctx, cell)
if err != nil {
results <- fmt.Errorf("TopologyServer.GetTabletsByCell(%v) failed: %v", cell, err)
continue
}
for _, alias := range aliases {
wg.Add(1)
go func(alias *pb.TabletAlias) {
defer wg.Done()
if err := topo.Validate(ctx, wr.ts, alias); err != nil {
results <- fmt.Errorf("Validate(%v) failed: %v", alias, err)
} else {
wr.Logger().Infof("tablet %v is valid", alias)
}
}(alias)
}
}
}
// Validate all tablets in all discoverable cells, even if they are
// not in the replication graph.
func (wr *Wrangler) validateAllTablets(wg *sync.WaitGroup, results chan<- vresult) {
cellSet := make(map[string]bool, 16)
keyspaces, err := wr.ts.GetKeyspaces()
if err != nil {
results <- vresult{"TopologyServer.GetKeyspaces", err}
return
}
for _, keyspace := range keyspaces {
shards, err := wr.ts.GetShardNames(keyspace)
if err != nil {
results <- vresult{"TopologyServer.GetShardNames(" + keyspace + ")", err}
return
}
for _, shard := range shards {
aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, keyspace, shard)
if err != nil {
results <- vresult{"TopologyServer.FindAllTabletAliasesInShard(" + keyspace + "," + shard + ")", err}
return
}
for _, alias := range aliases {
cellSet[alias.Cell] = true
}
}
}
for cell := range cellSet {
aliases, err := wr.ts.GetTabletsByCell(cell)
if err != nil {
results <- vresult{"GetTabletsByCell(" + cell + ")", err}
} else {
for _, alias := range aliases {
wg.Add(1)
go func(alias topo.TabletAlias) {
results <- vresult{alias.String(), topo.Validate(wr.ts, alias)}
wg.Done()
}(alias)
}
}
}
}
func (agent *ActionAgent) verifyTopology(ctx context.Context) {
if err := topo.Validate(ctx, agent.TopoServer, agent.TabletAlias); err != nil {
// Don't stop, it's not serious enough, this is likely transient.
log.Warningf("tablet validate failed: %v %v", agent.TabletAlias, err)
}
}