func (zkts *Server) CreateKeyspace(keyspace string, value *topo.Keyspace) error { keyspacePath := path.Join(globalKeyspacesPath, keyspace) pathList := []string{ keyspacePath, path.Join(keyspacePath, "action"), path.Join(keyspacePath, "actionlog"), path.Join(keyspacePath, "shards"), } alreadyExists := false for i, zkPath := range pathList { c := "" if i == 0 { c = jscfg.ToJson(value) } _, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { alreadyExists = true } else { return fmt.Errorf("error creating keyspace: %v %v", zkPath, err) } } } if alreadyExists { return topo.ErrNodeExists } event.Dispatch(&events.KeyspaceChange{ KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, value, -1), Status: "created", }) return nil }
func (zkts *Server) DeleteKeyspaceShards(keyspace string) error { shardsPath := path.Join(globalKeyspacesPath, keyspace, "shards") if err := zk.DeleteRecursive(zkts.zconn, shardsPath, -1); err != nil && !zookeeper.IsError(err, zookeeper.ZNONODE) { return err } event.Dispatch(&events.KeyspaceChange{ KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, nil, -1), Status: "deleted all shards", }) return nil }
func TestKeyspaceChangeSyslog(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-123 [keyspace] status" kc := &KeyspaceChange{ KeyspaceInfo: *topo.NewKeyspaceInfo("keyspace-123", nil, -1), Status: "status", } gotSev, gotMsg := kc.Syslog() if gotSev != wantSev { t.Errorf("wrong severity: got %v, want %v", gotSev, wantSev) } if gotMsg != wantMsg { t.Errorf("wrong message: got %v, want %v", gotMsg, wantMsg) } }
func (zkts *Server) GetKeyspace(keyspace string) (*topo.KeyspaceInfo, error) { keyspacePath := path.Join(globalKeyspacesPath, keyspace) data, stat, err := zkts.zconn.Get(keyspacePath) if err != nil { if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } return nil, err } k := &topo.Keyspace{} if err = json.Unmarshal([]byte(data), k); err != nil { return nil, fmt.Errorf("bad keyspace data %v", err) } return topo.NewKeyspaceInfo(keyspace, k, int64(stat.Version())), nil }
func TestMigrateServedFromSyslogReverse(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-1 [migrate served-from keyspace-2/source-shard <- keyspace-1/dest-shard] status" ev := &MigrateServedFrom{ Keyspace: *topo.NewKeyspaceInfo("keyspace-1", nil, -1), SourceShard: *topo.NewShardInfo("keyspace-2", "source-shard", nil, -1), DestinationShard: *topo.NewShardInfo("keyspace-1", "dest-shard", nil, -1), Reverse: true, StatusUpdater: base.StatusUpdater{Status: "status"}, } gotSev, gotMsg := ev.Syslog() if gotSev != wantSev { t.Errorf("wrong severity: got %v, want %v", gotSev, wantSev) } if gotMsg != wantMsg { t.Errorf("wrong message: got %v, want %v", gotMsg, wantMsg) } }
func TestMigrateServedTypesSyslogReverse(t *testing.T) { wantSev, wantMsg := syslog.LOG_INFO, "keyspace-1 [migrate served-types {src1, src2} <- {dst1, dst2}] status" ev := &MigrateServedTypes{ Keyspace: *topo.NewKeyspaceInfo("keyspace-1", nil, -1), SourceShards: []*topo.ShardInfo{ topo.NewShardInfo("keyspace-1", "src1", nil, -1), topo.NewShardInfo("keyspace-1", "src2", nil, -1), }, DestinationShards: []*topo.ShardInfo{ topo.NewShardInfo("keyspace-1", "dst1", nil, -1), topo.NewShardInfo("keyspace-1", "dst2", nil, -1), }, Reverse: true, StatusUpdater: base.StatusUpdater{Status: "status"}, } gotSev, gotMsg := ev.Syslog() if gotSev != wantSev { t.Errorf("wrong severity: got %v, want %v", gotSev, wantSev) } if gotMsg != wantMsg { t.Errorf("wrong message: got %v, want %v", gotMsg, wantMsg) } }
// migrateServedTypes operates with all concerned shards locked. func (wr *Wrangler) migrateServedTypes(keyspace string, sourceShards, destinationShards []*topo.ShardInfo, cells []string, servedType topo.TabletType, reverse bool) (err error) { // re-read all the shards so we are up to date wr.Logger().Infof("Re-reading all shards") for i, si := range sourceShards { if sourceShards[i], err = wr.ts.GetShard(si.Keyspace(), si.ShardName()); err != nil { return err } } for i, si := range destinationShards { if destinationShards[i], err = wr.ts.GetShard(si.Keyspace(), si.ShardName()); err != nil { return err } } ev := &events.MigrateServedTypes{ Keyspace: *topo.NewKeyspaceInfo(keyspace, nil, -1), SourceShards: sourceShards, DestinationShards: destinationShards, ServedType: servedType, Reverse: reverse, } event.DispatchUpdate(ev, "start") defer func() { if err != nil { event.DispatchUpdate(ev, "failed: "+err.Error()) } }() // For master type migration, need to: // - switch the source shards to read-only by disabling query service // - gather all replication points // - wait for filtered replication to catch up before we continue // - disable filtered replication after the fact if servedType == topo.TYPE_MASTER { event.DispatchUpdate(ev, "disabling query service on all source masters") for _, si := range sourceShards { if err := si.UpdateDisableQueryService(topo.TYPE_MASTER, nil, true); err != nil { return err } if err := topo.UpdateShard(context.TODO(), wr.ts, si); err != nil { return err } } if err := wr.refreshMasters(sourceShards); err != nil { return err } event.DispatchUpdate(ev, "getting positions of source masters") masterPositions, err := wr.getMastersPosition(sourceShards) if err != nil { return err } event.DispatchUpdate(ev, "waiting for destination masters to catch up") if err := wr.waitForFilteredReplication(masterPositions, destinationShards); err != nil { return err } for _, si := range destinationShards { si.SourceShards = nil } } // Check and update all shard records, in memory only. // We remember if we need to refresh the state of the source tablets // so their query service is enabled again, for reverse migration. needToRefreshSourceTablets := false for _, si := range sourceShards { if err := si.UpdateServedTypesMap(servedType, cells, !reverse); err != nil { return err } if tc, ok := si.TabletControlMap[servedType]; reverse && ok && tc.DisableQueryService { // this is a backward migration, where the // source tablets were disabled previously, so // we need to refresh them if err := si.UpdateDisableQueryService(servedType, cells, false); err != nil { return err } needToRefreshSourceTablets = true } if !reverse && servedType != topo.TYPE_MASTER { // this is a forward migration, we need to disable // query service on the source shards. // (this was already done for masters earlier) if err := si.UpdateDisableQueryService(servedType, cells, true); err != nil { return err } } } for _, si := range destinationShards { if err := si.UpdateServedTypesMap(servedType, cells, reverse); err != nil { return err } } // All is good, we can save the shards now event.DispatchUpdate(ev, "updating source shards") for _, si := range sourceShards { if err := topo.UpdateShard(context.TODO(), wr.ts, si); err != nil { return err } } if needToRefreshSourceTablets { event.DispatchUpdate(ev, "refreshing source shard tablets so they restart their query service") for _, si := range sourceShards { wr.RefreshTablesByShard(si, servedType, cells) } } event.DispatchUpdate(ev, "updating destination shards") for _, si := range destinationShards { if err := topo.UpdateShard(context.TODO(), wr.ts, si); err != nil { return err } } // And tell the new shards masters they can now be read-write. // Invoking a remote action will also make the tablet stop filtered // replication. if servedType == topo.TYPE_MASTER { event.DispatchUpdate(ev, "setting destination masters read-write") if err := wr.refreshMasters(destinationShards); err != nil { return err } } event.DispatchUpdate(ev, "finished") return nil }