// CopyKeyspaces will create the keyspaces in the destination topo
func CopyKeyspaces(fromTS, toTS topo.Server) {
keyspaces, err := fromTS.GetKeyspaces()
if err != nil {
log.Fatalf("GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
k, err := fromTS.GetKeyspace(keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetKeyspace(%v): %v", keyspace, err))
return
}
if err := toTS.CreateKeyspace(keyspace, k.Keyspace); err != nil {
if err == topo.ErrNodeExists {
log.Warningf("keyspace %v already exists", keyspace)
} else {
rec.RecordError(fmt.Errorf("CreateKeyspace(%v): %v", keyspace, err))
}
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyKeyspaces failed: %v", rec.Error())
}
}
// CopyTablets will create the tablets in the destination topo
func CopyTablets(fromTS, toTS topo.Server) {
cells, err := fromTS.GetKnownCells()
if err != nil {
log.Fatalf("fromTS.GetKnownCells failed: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range cells {
wg.Add(1)
go func(cell string) {
defer wg.Done()
tabletAliases, err := fromTS.GetTabletsByCell(cell)
if err != nil {
rec.RecordError(err)
} else {
for _, tabletAlias := range tabletAliases {
wg.Add(1)
go func(tabletAlias topo.TabletAlias) {
defer wg.Done()
// read the source tablet
ti, err := fromTS.GetTablet(tabletAlias)
if err != nil {
rec.RecordError(err)
return
}
// try to create the destination
err = toTS.CreateTablet(ti.Tablet)
if err == topo.ErrNodeExists {
// update the destination tablet
log.Warningf("tablet %v already exists, updating it", tabletAlias)
err = toTS.UpdateTabletFields(ti.Alias(), func(t *topo.Tablet) error {
*t = *ti.Tablet
return nil
})
}
if err != nil {
rec.RecordError(err)
return
}
// create the replication paths
// for masters only here
if ti.Type == topo.TYPE_MASTER {
if err = toTS.CreateReplicationPath(ti.Keyspace, ti.Shard, ti.Alias().String()); err != nil && err != topo.ErrNodeExists {
rec.RecordError(err)
}
}
}(tabletAlias)
}
}
}(cell)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyTablets failed: %v", rec.Error())
}
}
// FindAllTabletAliasesInShardByCell uses the replication graph to find all the
// tablet aliases in the given shard.
//
// It can return ErrPartialResult if some cells were not fetched,
// in which case the result only contains the cells that were fetched.
//
// The tablet aliases are sorted by cell, then by UID.
func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) ([]TabletAlias, error) {
span := trace.NewSpanFromContext(ctx)
span.StartLocal("topo.FindAllTabletAliasesInShardbyCell")
span.Annotate("keyspace", keyspace)
span.Annotate("shard", shard)
span.Annotate("num_cells", len(cells))
defer span.Finish()
ctx = trace.NewContext(ctx, span)
// read the shard information to find the cells
si, err := GetShard(ctx, ts, keyspace, shard)
if err != nil {
return nil, err
}
resultAsMap := make(map[TabletAlias]bool)
if si.MasterAlias != nil && !TabletAliasIsZero(si.MasterAlias) {
if InCellList(si.MasterAlias.Cell, cells) {
resultAsMap[ProtoToTabletAlias(si.MasterAlias)] = true
}
}
// read the replication graph in each cell and add all found tablets
wg := sync.WaitGroup{}
mutex := sync.Mutex{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range si.Cells {
if !InCellList(cell, cells) {
continue
}
wg.Add(1)
go func(cell string) {
defer wg.Done()
sri, err := ts.GetShardReplication(ctx, cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err))
return
}
mutex.Lock()
for _, node := range sri.Nodes {
resultAsMap[ProtoToTabletAlias(node.TabletAlias)] = true
}
mutex.Unlock()
}(cell)
}
wg.Wait()
err = nil
if rec.HasErrors() {
log.Warningf("FindAllTabletAliasesInShard(%v,%v): got partial result: %v", keyspace, shard, rec.Error())
err = ErrPartialResult
}
result := make([]TabletAlias, 0, len(resultAsMap))
for a := range resultAsMap {
result = append(result, a)
}
sort.Sort(TabletAliasList(result))
return result, err
}
// DiffSchemaToArray diffs two schemas and return the schema diffs if there is any.
func DiffSchemaToArray(leftName string, left *SchemaDefinition, rightName string, right *SchemaDefinition) (result []string) {
er := concurrency.AllErrorRecorder{}
DiffSchema(leftName, left, rightName, right, &er)
if er.HasErrors() {
return er.ErrorStrings()
}
return nil
}
// DiffPermissionsToArray difs two sets of permissions, and returns the difference
func DiffPermissionsToArray(leftName string, left *tabletmanagerdatapb.Permissions, rightName string, right *tabletmanagerdatapb.Permissions) (result []string) {
er := concurrency.AllErrorRecorder{}
DiffPermissions(leftName, left, rightName, right, &er)
if er.HasErrors() {
return er.ErrorStrings()
}
return nil
}
// Make this external, since these transitions need to be forced from time to time.
func ChangeType(ts topo.Server, tabletAlias topo.TabletAlias, newType topo.TabletType, runHooks bool) error {
tablet, err := ts.GetTablet(tabletAlias)
if err != nil {
return err
}
if !topo.IsTrivialTypeChange(tablet.Type, newType) || !topo.IsValidTypeChange(tablet.Type, newType) {
return fmt.Errorf("cannot change tablet type %v -> %v %v", tablet.Type, newType, tabletAlias)
}
if runHooks {
// Only run the preflight_serving_type hook when
// transitioning from non-serving to serving.
if !topo.IsInServingGraph(tablet.Type) && topo.IsInServingGraph(newType) {
if err := hook.NewSimpleHook("preflight_serving_type").ExecuteOptional(); err != nil {
return err
}
}
}
tablet.Type = newType
if newType == topo.TYPE_IDLE {
if tablet.Parent.IsZero() {
si, err := ts.GetShard(tablet.Keyspace, tablet.Shard)
if err != nil {
return err
}
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, cell := range si.Cells {
wg.Add(1)
go func(cell string) {
defer wg.Done()
sri, err := ts.GetShardReplication(cell, tablet.Keyspace, tablet.Shard)
if err != nil {
log.Warningf("Cannot check cell %v for extra replication paths, assuming it's good", cell)
return
}
for _, rl := range sri.ReplicationLinks {
if rl.Parent == tabletAlias {
rec.RecordError(fmt.Errorf("Still have a ReplicationLink in cell %v", cell))
}
}
}(cell)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
}
tablet.Parent = topo.TabletAlias{}
tablet.Keyspace = ""
tablet.Shard = ""
tablet.KeyRange = key.KeyRange{}
}
return topo.UpdateTablet(ts, tablet)
}
// shardsWithTablesSources returns all the shards that have SourceShards set
// to one value, with an array of Tables.
func shardsWithTablesSources(ctx context.Context, wr *wrangler.Wrangler) ([]map[string]string, error) {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
keyspaces, err := wr.TopoServer().GetKeyspaces(shortCtx)
cancel()
if err != nil {
return nil, err
}
wg := sync.WaitGroup{}
mu := sync.Mutex{} // protects result
result := make([]map[string]string, 0, len(keyspaces))
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
shards, err := wr.TopoServer().GetShardNames(shortCtx, keyspace)
cancel()
if err != nil {
rec.RecordError(err)
return
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
si, err := wr.TopoServer().GetShard(shortCtx, keyspace, shard)
cancel()
if err != nil {
rec.RecordError(err)
return
}
if len(si.SourceShards) == 1 && len(si.SourceShards[0].Tables) > 0 {
mu.Lock()
result = append(result, map[string]string{
"Keyspace": keyspace,
"Shard": shard,
})
mu.Unlock()
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
return nil, rec.Error()
}
if len(result) == 0 {
return nil, fmt.Errorf("There are no shards with SourceShards")
}
return result, nil
}
func DiffPermissionsToArray(leftName string, left *Permissions, rightName string, right *Permissions) (result []string) {
er := concurrency.AllErrorRecorder{}
DiffPermissions(leftName, left, rightName, right, &er)
if er.HasErrors() {
return er.Errors
} else {
return nil
}
}
func (wr *Wrangler) ValidatePermissionsKeyspace(keyspace string) error {
// find all the shards
shards, err := wr.ts.GetShardNames(keyspace)
if err != nil {
return err
}
// corner cases
if len(shards) == 0 {
return fmt.Errorf("No shards in keyspace %v", keyspace)
}
sort.Strings(shards)
if len(shards) == 1 {
return wr.ValidatePermissionsShard(keyspace, shards[0])
}
// find the reference permissions using the first shard's master
si, err := wr.ts.GetShard(keyspace, shards[0])
if err != nil {
return err
}
if si.MasterAlias.Uid == topo.NO_TABLET {
return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0])
}
referenceAlias := si.MasterAlias
log.Infof("Gathering permissions for reference master %v", referenceAlias)
referencePermissions, err := wr.GetPermissions(si.MasterAlias)
if err != nil {
return err
}
// then diff with all tablets but master 0
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, shard := range shards {
aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, keyspace, shard)
if err != nil {
er.RecordError(err)
continue
}
for _, alias := range aliases {
if alias == si.MasterAlias {
continue
}
wg.Add(1)
go wr.diffPermissions(referencePermissions, referenceAlias, alias, &wg, &er)
}
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Permissions diffs:\n%v", er.Error().Error())
}
return nil
}
// CopyShardReplications will create the ShardReplication objects in
// the destination topo
func CopyShardReplications(ctx context.Context, fromTS, toTS topo.Impl) {
keyspaces, err := fromTS.GetKeyspaces(ctx)
if err != nil {
log.Fatalf("fromTS.GetKeyspaces: %v", err)
}
tts := topo.Server{
Impl: toTS,
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shards, err := fromTS.GetShardNames(ctx, keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err))
return
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
// read the source shard to get the cells
s, _, err := fromTS.GetShard(ctx, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err))
return
}
for _, cell := range s.Cells {
sri, err := fromTS.GetShardReplication(ctx, cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v): %v", cell, keyspace, shard, err))
continue
}
if err := tts.UpdateShardReplicationFields(ctx, cell, keyspace, shard, func(oldSR *topodatapb.ShardReplication) error {
*oldSR = *sri.ShardReplication
return nil
}); err != nil {
rec.RecordError(fmt.Errorf("UpdateShardReplicationFields(%v, %v, %v): %v", cell, keyspace, shard, err))
}
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyShards failed: %v", rec.Error())
}
}
// CopyShards will create the shards in the destination topo
func CopyShards(fromTS, toTS topo.Server, deleteKeyspaceShards bool) {
keyspaces, err := fromTS.GetKeyspaces()
if err != nil {
log.Fatalf("fromTS.GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shards, err := fromTS.GetShardNames(keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err))
return
}
if deleteKeyspaceShards {
if err := toTS.DeleteKeyspaceShards(keyspace); err != nil {
rec.RecordError(fmt.Errorf("DeleteKeyspaceShards(%v): %v", keyspace, err))
return
}
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
if err := topo.CreateShard(toTS, keyspace, shard); err != nil {
if err == topo.ErrNodeExists {
log.Warningf("shard %v/%v already exists", keyspace, shard)
} else {
rec.RecordError(fmt.Errorf("CreateShard(%v, %v): %v", keyspace, shard, err))
return
}
}
si, err := fromTS.GetShard(keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err))
return
}
if err := toTS.UpdateShard(si); err != nil {
rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err))
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyShards failed: %v", rec.Error())
}
}
// ValidateVersionKeyspace validates all versions are the same in all
// tablets in a keyspace
func (wr *Wrangler) ValidateVersionKeyspace(ctx context.Context, keyspace string) error {
// find all the shards
shards, err := wr.ts.GetShardNames(ctx, keyspace)
if err != nil {
return err
}
// corner cases
if len(shards) == 0 {
return fmt.Errorf("No shards in keyspace %v", keyspace)
}
sort.Strings(shards)
if len(shards) == 1 {
return wr.ValidateVersionShard(ctx, keyspace, shards[0])
}
// find the reference version using the first shard's master
si, err := wr.ts.GetShard(ctx, keyspace, shards[0])
if err != nil {
return err
}
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shards[0])
}
referenceAlias := si.MasterAlias
log.Infof("Gathering version for reference master %v", topo.TabletAliasString(referenceAlias))
referenceVersion, err := wr.GetVersion(ctx, referenceAlias)
if err != nil {
return err
}
// then diff with all tablets but master 0
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, shard := range shards {
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
er.RecordError(err)
continue
}
for _, alias := range aliases {
if topo.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffVersion(ctx, referenceVersion, referenceAlias, alias, &wg, &er)
}
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Version diffs:\n%v", er.Error().Error())
}
return nil
}
// CopyShards will create the shards in the destination topo
func CopyShards(ctx context.Context, fromTS, toTS topo.Impl) {
keyspaces, err := fromTS.GetKeyspaces(ctx)
if err != nil {
log.Fatalf("fromTS.GetKeyspaces: %v", err)
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shards, err := fromTS.GetShardNames(ctx, keyspace)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardNames(%v): %v", keyspace, err))
return
}
for _, shard := range shards {
wg.Add(1)
go func(keyspace, shard string) {
defer wg.Done()
if err := toTS.CreateShard(ctx, keyspace, shard, &topodatapb.Shard{}); err != nil {
if err == topo.ErrNodeExists {
log.Warningf("shard %v/%v already exists", keyspace, shard)
} else {
rec.RecordError(fmt.Errorf("CreateShard(%v, %v): %v", keyspace, shard, err))
return
}
}
s, _, err := fromTS.GetShard(ctx, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShard(%v, %v): %v", keyspace, shard, err))
return
}
_, toV, err := toTS.GetShard(ctx, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("toTS.GetShard(%v, %v): %v", keyspace, shard, err))
return
}
if _, err := toTS.UpdateShard(ctx, keyspace, shard, s, toV); err != nil {
rec.RecordError(fmt.Errorf("UpdateShard(%v, %v): %v", keyspace, shard, err))
}
}(keyspace, shard)
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyShards failed: %v", rec.Error())
}
}
// CopyTablets will create the tablets in the destination topo
func CopyTablets(ctx context.Context, fromTS, toTS topo.Impl) {
cells, err := fromTS.GetKnownCells(ctx)
if err != nil {
log.Fatalf("fromTS.GetKnownCells: %v", err)
}
tts := topo.Server{
Impl: toTS,
}
wg := sync.WaitGroup{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range cells {
wg.Add(1)
go func(cell string) {
defer wg.Done()
tabletAliases, err := fromTS.GetTabletsByCell(ctx, cell)
if err != nil {
rec.RecordError(fmt.Errorf("GetTabletsByCell(%v): %v", cell, err))
} else {
for _, tabletAlias := range tabletAliases {
wg.Add(1)
go func(tabletAlias *topodatapb.TabletAlias) {
defer wg.Done()
// read the source tablet
tablet, _, err := fromTS.GetTablet(ctx, tabletAlias)
if err != nil {
rec.RecordError(fmt.Errorf("GetTablet(%v): %v", tabletAlias, err))
return
}
// try to create the destination
err = toTS.CreateTablet(ctx, tablet)
if err == topo.ErrNodeExists {
// update the destination tablet
log.Warningf("tablet %v already exists, updating it", tabletAlias)
_, err = tts.UpdateTabletFields(ctx, tablet.Alias, func(t *topodatapb.Tablet) error {
*t = *tablet
return nil
})
}
if err != nil {
rec.RecordError(fmt.Errorf("CreateTablet(%v): %v", tabletAlias, err))
return
}
}(tabletAlias)
}
}
}(cell)
}
wg.Wait()
if rec.HasErrors() {
log.Fatalf("copyTablets failed: %v", rec.Error())
}
}
// FindAllTabletAliasesInShard uses the replication graph to find all the
// tablet aliases in the given shard.
// It can return ErrPartialResult if some cells were not fetched,
// in which case the result only contains the cells that were fetched.
func FindAllTabletAliasesInShardByCell(ts Server, keyspace, shard string, cells []string) ([]TabletAlias, error) {
// read the shard information to find the cells
si, err := ts.GetShard(keyspace, shard)
if err != nil {
return nil, err
}
resultAsMap := make(map[TabletAlias]bool)
if !si.MasterAlias.IsZero() {
if InCellList(si.MasterAlias.Cell, cells) {
resultAsMap[si.MasterAlias] = true
}
}
// read the replication graph in each cell and add all found tablets
wg := sync.WaitGroup{}
mutex := sync.Mutex{}
rec := concurrency.AllErrorRecorder{}
for _, cell := range si.Cells {
if !InCellList(cell, cells) {
continue
}
wg.Add(1)
go func(cell string) {
defer wg.Done()
sri, err := ts.GetShardReplication(cell, keyspace, shard)
if err != nil {
rec.RecordError(fmt.Errorf("GetShardReplication(%v, %v, %v) failed: %v", cell, keyspace, shard, err))
return
}
mutex.Lock()
for _, rl := range sri.ReplicationLinks {
resultAsMap[rl.TabletAlias] = true
if !rl.Parent.IsZero() && InCellList(rl.Parent.Cell, cells) {
resultAsMap[rl.Parent] = true
}
}
mutex.Unlock()
}(cell)
}
wg.Wait()
err = nil
if rec.HasErrors() {
log.Warningf("FindAllTabletAliasesInShard(%v,%v): got partial result: %v", keyspace, shard, rec.Error())
err = ErrPartialResult
}
result := make([]TabletAlias, 0, len(resultAsMap))
for a := range resultAsMap {
result = append(result, a)
}
return result, err
}
// Execute executes a non-streaming query on the specified shards.
func (stc *ScatterConn) ExecuteBatch(queries []proto.BoundQuery, keyspace string, shards []string) (qrs *tproto.QueryResultList, err error) {
stc.mu.Lock()
defer stc.mu.Unlock()
qrs = &tproto.QueryResultList{List: make([]mproto.QueryResult, len(queries))}
allErrors := new(concurrency.AllErrorRecorder)
if len(shards) == 0 {
return qrs, nil
}
results := make(chan *tproto.QueryResultList, len(shards))
var wg sync.WaitGroup
for shard := range unique(shards) {
wg.Add(1)
go func(shard string) {
defer wg.Done()
sdc, err := stc.getConnection(keyspace, shard)
if err != nil {
allErrors.RecordError(err)
return
}
innerqrs, err := sdc.ExecuteBatch(queries)
if err != nil {
allErrors.RecordError(err)
return
}
results <- innerqrs
}(shard)
}
go func() {
wg.Wait()
close(results)
}()
for innerqr := range results {
for i := range qrs.List {
appendResult(&qrs.List[i], &innerqr.List[i])
}
}
if allErrors.HasErrors() {
if stc.transactionId != 0 {
errstr := allErrors.Error().Error()
// We cannot recover from these errors
if strings.Contains(errstr, "tx_pool_full") || strings.Contains(errstr, "not_in_tx") {
stc.rollback()
}
}
return nil, allErrors.Error()
}
return qrs, nil
}
// Execute executes a non-streaming query on the specified shards.
func (stc *ScatterConn) Execute(query string, bindVars map[string]interface{}, keyspace string, shards []string) (*mproto.QueryResult, error) {
stc.mu.Lock()
defer stc.mu.Unlock()
qr := new(mproto.QueryResult)
allErrors := new(concurrency.AllErrorRecorder)
switch len(shards) {
case 0:
return qr, nil
case 1:
// Fast-path for single shard execution
var err error
qr, err = stc.execOnShard(query, bindVars, keyspace, shards[0])
allErrors.RecordError(err)
default:
results := make(chan *mproto.QueryResult, len(shards))
var wg sync.WaitGroup
for shard := range unique(shards) {
wg.Add(1)
go func(shard string) {
defer wg.Done()
innerqr, err := stc.execOnShard(query, bindVars, keyspace, shard)
if err != nil {
allErrors.RecordError(err)
return
}
results <- innerqr
}(shard)
}
go func() {
wg.Wait()
close(results)
}()
for innerqr := range results {
appendResult(qr, innerqr)
}
}
if allErrors.HasErrors() {
if stc.transactionId != 0 {
errstr := allErrors.Error().Error()
// We cannot recover from these errors
if strings.Contains(errstr, "tx_pool_full") || strings.Contains(errstr, "not_in_tx") {
stc.rollback()
}
}
return nil, allErrors.Error()
}
return qr, nil
}
// initializeSwap starts the schema swap process. If there is already a schema swap process started
// the the method just picks up that. Otherwise it starts a new one and writes into the database that
// the process was started.
func (schemaSwap *Swap) initializeSwap(sql string) error {
var waitGroup sync.WaitGroup
metadataList := make([]shardSwapMetadata, len(schemaSwap.allShards))
for i, shard := range schemaSwap.allShards {
waitGroup.Add(1)
go shard.readShardMetadata(&metadataList[i], &waitGroup)
}
waitGroup.Wait()
var recorder concurrency.AllErrorRecorder
for i, metadata := range metadataList {
if metadata.err != nil {
recorder.RecordError(metadata.err)
} else if metadata.lastStartedSwap == metadata.lastFinishedSwap {
// The shard doesn't have schema swap started yet.
nextSwapID := metadata.lastFinishedSwap + 1
if schemaSwap.swapID == 0 {
schemaSwap.swapID = nextSwapID
} else if schemaSwap.swapID != nextSwapID {
recorder.RecordError(fmt.Errorf(
"Next schema swap id on shard %v should be %v, while for other shard(s) it should be %v",
schemaSwap.allShards[i].shardName, nextSwapID, schemaSwap.swapID))
}
} else if metadata.lastStartedSwap < metadata.lastFinishedSwap {
recorder.RecordError(fmt.Errorf(
"Bad swap metadata on shard %v: LastFinishedSchemaSwap=%v is greater than LastStartedSchemaSwap=%v",
schemaSwap.allShards[i].shardName, metadata.lastFinishedSwap, metadata.lastStartedSwap))
} else if schemaSwap.swapID != 0 && schemaSwap.swapID != metadata.lastStartedSwap {
recorder.RecordError(fmt.Errorf(
"Shard %v has an already started schema swap with an id %v, while for other shard(s) id should be equal to %v",
schemaSwap.allShards[i].shardName, metadata.lastStartedSwap, schemaSwap.swapID))
} else if metadata.currentSQL != sql {
recorder.RecordError(fmt.Errorf(
"Shard %v has an already started schema swap with a different set of SQL statements",
schemaSwap.allShards[i].shardName))
} else {
schemaSwap.swapID = metadata.lastStartedSwap
}
}
if recorder.HasErrors() {
return recorder.Error()
}
return schemaSwap.runOnAllShards(
func(shard *shardSchemaSwap) error {
return shard.writeStartedSwap(sql)
})
}
// InitializeConnections pre-initializes all ShardConn which create underlying connections.
// It also populates topology cache by accessing it.
// It is not necessary to call this function before serving queries,
// but it would reduce connection overhead when serving.
func (stc *ScatterConn) InitializeConnections(ctx context.Context) error {
ksNames, err := stc.toposerv.GetSrvKeyspaceNames(ctx, stc.cell)
if err != nil {
return err
}
var wg sync.WaitGroup
var errRecorder concurrency.AllErrorRecorder
for _, ksName := range ksNames {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
// get SrvKeyspace for cell/keyspace
ks, err := stc.toposerv.GetSrvKeyspace(ctx, stc.cell, keyspace)
if err != nil {
errRecorder.RecordError(err)
return
}
// work on all shards of all serving tablet types
for _, tabletType := range ks.TabletTypes {
ksPartition, ok := ks.Partitions[tabletType]
if !ok {
errRecorder.RecordError(fmt.Errorf("%v.%v is not in SrvKeyspace.Partitions", keyspace, string(tabletType)))
continue
}
for _, shard := range ksPartition.Shards {
wg.Add(1)
go func(shardName string, tabletType topo.TabletType) {
defer wg.Done()
shardConn := stc.getConnection(ctx, keyspace, shardName, tabletType)
err = shardConn.Dial(ctx)
if err != nil {
errRecorder.RecordError(err)
return
}
}(shard.ShardName(), tabletType)
}
}
}(ksName)
}
wg.Wait()
if errRecorder.HasErrors() {
return errRecorder.Error()
}
return nil
}
// ExecuteBatch executes a batch of non-streaming queries on the specified shards.
func (stc *ScatterConn) ExecuteBatch(
ctx context.Context,
batchRequest *scatterBatchRequest,
tabletType topodatapb.TabletType,
asTransaction bool,
session *SafeSession) (qrs []sqltypes.Result, err error) {
allErrors := new(concurrency.AllErrorRecorder)
results := make([]sqltypes.Result, batchRequest.Length)
var resMutex sync.Mutex
var wg sync.WaitGroup
for _, req := range batchRequest.Requests {
wg.Add(1)
go func(req *shardBatchRequest) {
defer wg.Done()
startTime, statsKey, transactionID, err := stc.startAction(ctx, "ExecuteBatch", req.Keyspace, req.Shard, tabletType, session, false, allErrors)
defer stc.endAction(startTime, allErrors, statsKey, &err)
if err != nil {
return
}
var innerqrs []sqltypes.Result
innerqrs, err = stc.gateway.ExecuteBatch(ctx, req.Keyspace, req.Shard, tabletType, req.Queries, asTransaction, transactionID)
if err != nil {
return
}
resMutex.Lock()
defer resMutex.Unlock()
for i, result := range innerqrs {
appendResult(&results[req.ResultIndexes[i]], &result)
}
}(req)
}
wg.Wait()
// If we want to rollback, we have to do it before closing results
// so that the session is updated to be not InTransaction.
if allErrors.HasErrors() {
stc.rollbackIfNeeded(ctx, allErrors, session)
return nil, allErrors.AggrError(stc.aggregateErrors)
}
return results, nil
}
func keyspacesWithOverlappingShards(ctx context.Context, wr *wrangler.Wrangler) ([]map[string]string, error) {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
keyspaces, err := wr.TopoServer().GetKeyspaces(shortCtx)
cancel()
if err != nil {
return nil, fmt.Errorf("failed to get list of keyspaces: %v", err)
}
wg := sync.WaitGroup{}
mu := sync.Mutex{} // protects result
result := make([]map[string]string, 0, len(keyspaces))
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shortCtx, cancel = context.WithTimeout(ctx, *remoteActionsTimeout)
osList, err := topotools.FindOverlappingShards(shortCtx, wr.TopoServer(), keyspace)
cancel()
if err != nil {
rec.RecordError(err)
return
}
mu.Lock()
for _, os := range osList {
result = append(result, map[string]string{
"Keyspace": os.Left[0].Keyspace(),
"Shard": os.Left[0].ShardName(),
})
}
mu.Unlock()
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
return nil, rec.Error()
}
if len(result) == 0 {
return nil, fmt.Errorf("There are no keyspaces with overlapping shards")
}
return result, nil
}
// findAllKeyspaceShards goes through all serving shards in the topology
func findAllKeyspaceShards(ctx context.Context, ts topo.SrvTopoServer, cell string) (map[keyspaceShard]bool, error) {
ksNames, err := ts.GetSrvKeyspaceNames(ctx, cell)
if err != nil {
return nil, err
}
keyspaceShards := make(map[keyspaceShard]bool)
var wg sync.WaitGroup
var mu sync.Mutex
var errRecorder concurrency.AllErrorRecorder
for _, ksName := range ksNames {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
// get SrvKeyspace for cell/keyspace
ks, err := ts.GetSrvKeyspace(ctx, cell, keyspace)
if err != nil {
errRecorder.RecordError(err)
return
}
// get all shard names that are used for serving
mu.Lock()
for _, ksPartition := range ks.Partitions {
for _, shard := range ksPartition.ShardReferences {
keyspaceShards[keyspaceShard{
keyspace: keyspace,
shard: shard.Name,
}] = true
}
}
mu.Unlock()
}(ksName)
}
wg.Wait()
if errRecorder.HasErrors() {
return nil, errRecorder.Error()
}
return keyspaceShards, nil
}
// InitializeConnections pre-initializes connections for all shards.
// It also populates topology cache by accessing it.
// It is not necessary to call this function before serving queries,
// but it would reduce connection overhead when serving.
func (sg *shardGateway) InitializeConnections(ctx context.Context) error {
ksNames, err := sg.toposerv.GetSrvKeyspaceNames(ctx, sg.cell)
if err != nil {
return err
}
var wg sync.WaitGroup
var errRecorder concurrency.AllErrorRecorder
for _, ksName := range ksNames {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
// get SrvKeyspace for cell/keyspace
ks, err := sg.toposerv.GetSrvKeyspace(ctx, sg.cell, keyspace)
if err != nil {
errRecorder.RecordError(err)
return
}
// work on all shards of all serving tablet types
for _, ksPartition := range ks.Partitions {
tt := ksPartition.ServedType
for _, shard := range ksPartition.ShardReferences {
wg.Add(1)
go func(shardName string, tabletType topodatapb.TabletType) {
defer wg.Done()
err = sg.getConnection(ctx, keyspace, shardName, tabletType).Dial(ctx)
if err != nil {
errRecorder.RecordError(err)
return
}
}(shard.Name, tt)
}
}
}(ksName)
}
wg.Wait()
if errRecorder.HasErrors() {
return errRecorder.AggrError(AggregateVtGateErrors)
}
return nil
}
// ValidateSchemaShard will diff the schema from all the tablets in the shard.
func (wr *Wrangler) ValidateSchemaShard(ctx context.Context, keyspace, shard string, excludeTables []string, includeViews bool) error {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// get schema from the master, or error
if !si.HasMaster() {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
log.Infof("Gathering schema for master %v", topoproto.TabletAliasString(si.MasterAlias))
masterSchema, err := wr.GetSchema(ctx, si.MasterAlias, nil, excludeTables, includeViews)
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
if err != nil {
return err
}
// then diff with all slaves
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if topoproto.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffSchema(ctx, masterSchema, si.MasterAlias, alias, excludeTables, includeViews, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Schema diffs: %v", er.Error().Error())
}
return nil
}
func (wr *Wrangler) ValidateSchemaShard(keyspace, shard string, includeViews bool) error {
si, err := wr.ts.GetShard(keyspace, shard)
if err != nil {
return err
}
// get schema from the master, or error
if si.MasterAlias.Uid == topo.NO_TABLET {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
relog.Info("Gathering schema for master %v", si.MasterAlias)
masterSchema, err := wr.GetSchema(si.MasterAlias, nil, includeViews)
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := topo.FindAllTabletAliasesInShard(wr.ts, keyspace, shard)
if err != nil {
return err
}
// then diff with all slaves
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if alias == si.MasterAlias {
continue
}
wg.Add(1)
go wr.diffSchema(masterSchema, si.MasterAlias, alias, includeViews, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Schema diffs:\n%v", er.Error().Error())
}
return nil
}
// InitializeConnections pre-initializes connections for all shards.
// It also populates topology cache by accessing it.
// It is not necessary to call this function before serving queries,
// but it would reduce connection overhead when serving.
func (stc *ScatterConn) InitializeConnections(ctx context.Context) error {
ksNames, err := stc.toposerv.GetSrvKeyspaceNames(ctx, stc.cell)
if err != nil {
return err
}
var wg sync.WaitGroup
var errRecorder concurrency.AllErrorRecorder
for _, ksName := range ksNames {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
// get SrvKeyspace for cell/keyspace
ks, err := stc.toposerv.GetSrvKeyspace(ctx, stc.cell, keyspace)
if err != nil {
errRecorder.RecordError(err)
return
}
// work on all shards of all serving tablet types
for tabletType, ksPartition := range ks.Partitions {
tt := topo.TabletTypeToProto(tabletType)
for _, shard := range ksPartition.ShardReferences {
wg.Add(1)
go func(shardName string, tabletType pb.TabletType) {
defer wg.Done()
err = stc.gateway.Dial(ctx, keyspace, shardName, tabletType)
if err != nil {
errRecorder.RecordError(err)
return
}
}(shard.Name, tt)
}
}
}(ksName)
}
wg.Wait()
if errRecorder.HasErrors() {
return errRecorder.Error()
}
return nil
}
// keyspacesWithServedFrom returns all the keyspaces that have ServedFrom set
// to one value.
func keyspacesWithServedFrom(ctx context.Context, wr *wrangler.Wrangler) ([]string, error) {
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
keyspaces, err := wr.TopoServer().GetKeyspaces(shortCtx)
cancel()
if err != nil {
return nil, fmt.Errorf("failed to get list of keyspaces: %v", err)
}
wg := sync.WaitGroup{}
mu := sync.Mutex{} // protects result
result := make([]string, 0, len(keyspaces))
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
ki, err := wr.TopoServer().GetKeyspace(shortCtx, keyspace)
cancel()
if err != nil {
rec.RecordError(fmt.Errorf("failed to get details for keyspace '%v': %v", keyspace, err))
return
}
if len(ki.ServedFroms) > 0 {
mu.Lock()
result = append(result, keyspace)
mu.Unlock()
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
return nil, rec.Error()
}
if len(result) == 0 {
return nil, fmt.Errorf("there are no keyspaces with ServedFrom")
}
return result, nil
}
// ValidateVersionShard validates all versions are the same in all
// tablets in a shard
func (wr *Wrangler) ValidateVersionShard(ctx context.Context, keyspace, shard string) error {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// get version from the master, or error
if topo.TabletAliasIsZero(si.MasterAlias) {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
log.Infof("Gathering version for master %v", topo.TabletAliasString(si.MasterAlias))
masterVersion, err := wr.GetVersion(ctx, si.MasterAlias)
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shard)
if err != nil {
return err
}
// then diff with all slaves
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if topo.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffVersion(ctx, masterVersion, si.MasterAlias, alias, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Version diffs:\n%v", er.Error().Error())
}
return nil
}
// ValidatePermissionsShard validates all the permissions are the same
// in a shard
func (wr *Wrangler) ValidatePermissionsShard(ctx context.Context, keyspace, shard string) error {
si, err := wr.ts.GetShard(ctx, keyspace, shard)
if err != nil {
return err
}
// get permissions from the master, or error
if !si.HasMaster() {
return fmt.Errorf("No master in shard %v/%v", keyspace, shard)
}
log.Infof("Gathering permissions for master %v", topoproto.TabletAliasString(si.MasterAlias))
masterPermissions, err := wr.GetPermissions(ctx, si.MasterAlias)
if err != nil {
return err
}
// read all the aliases in the shard, that is all tablets that are
// replicating from the master
aliases, err := wr.ts.FindAllTabletAliasesInShard(ctx, keyspace, shard)
if err != nil {
return err
}
// then diff all of them, except master
er := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for _, alias := range aliases {
if topoproto.TabletAliasEqual(alias, si.MasterAlias) {
continue
}
wg.Add(1)
go wr.diffPermissions(ctx, masterPermissions, si.MasterAlias, alias, &wg, &er)
}
wg.Wait()
if er.HasErrors() {
return fmt.Errorf("Permissions diffs:\n%v", er.Error().Error())
}
return nil
}
// keyspacesWithServedFrom returns all the keyspaces that have ServedFrom set
// to one value.
func keyspacesWithServedFrom(ctx context.Context, wr *wrangler.Wrangler) ([]string, error) {
keyspaces, err := wr.TopoServer().GetKeyspaces(ctx)
if err != nil {
return nil, err
}
wg := sync.WaitGroup{}
mu := sync.Mutex{} // protects result
result := make([]string, 0, len(keyspaces))
rec := concurrency.AllErrorRecorder{}
for _, keyspace := range keyspaces {
wg.Add(1)
go func(keyspace string) {
defer wg.Done()
ki, err := wr.TopoServer().GetKeyspace(ctx, keyspace)
if err != nil {
rec.RecordError(err)
return
}
if len(ki.ServedFroms) > 0 {
mu.Lock()
result = append(result, keyspace)
mu.Unlock()
}
}(keyspace)
}
wg.Wait()
if rec.HasErrors() {
return nil, rec.Error()
}
if len(result) == 0 {
return nil, fmt.Errorf("There are no keyspaces with ServedFrom")
}
return result, nil
}