func WaitClusterDataWithMaster(e *store.StoreManager, timeout time.Duration) (string, error) {
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
if cd.Cluster.Status.Phase == cluster.ClusterPhaseNormal && cd.Cluster.Status.Master != "" {
return cd.DBs[cd.Cluster.Status.Master].Spec.KeeperUID, nil
}
end:
time.Sleep(2 * time.Second)
}
return "", fmt.Errorf("timeout")
}
func WaitClusterPhase(e *store.StoreManager, phase cluster.ClusterPhase, timeout time.Duration) error {
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
if cd.Cluster.Status.Phase == phase {
return nil
}
end:
time.Sleep(2 * time.Second)
}
return fmt.Errorf("timeout")
}
func WaitNumDBs(e *store.StoreManager, n int, timeout time.Duration) error {
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
if len(cd.DBs) == n {
return nil
}
end:
time.Sleep(sleepInterval)
}
return fmt.Errorf("timeout")
}
func getClusterData(e *store.StoreManager) (*cluster.ClusterData, *kvstore.KVPair, error) {
cd, pair, err := e.GetClusterData()
if err != nil {
return nil, nil, fmt.Errorf("cannot get cluster data: %v", err)
}
if cd == nil {
return nil, nil, fmt.Errorf("nil cluster data: %v", err)
}
if cd.FormatVersion != cluster.CurrentCDFormatVersion {
return nil, nil, fmt.Errorf("unsupported cluster data format version %d", cd.FormatVersion)
}
if err := cd.Cluster.Spec.Validate(); err != nil {
return nil, nil, fmt.Errorf("clusterdata validation failed: %v", err)
}
return cd, pair, nil
}
func WaitClusterDataMaster(master string, e *store.StoreManager, timeout time.Duration) error {
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
if cd.Cluster.Status.Phase == cluster.ClusterPhaseNormal && cd.Cluster.Status.Master != "" {
if cd.DBs[cd.Cluster.Status.Master].Spec.KeeperUID == master {
return nil
}
}
end:
time.Sleep(sleepInterval)
}
return fmt.Errorf("timeout")
}
// WaitClusterDataSynchronousStandbys waits for:
// * synchrnous standby defined in masterdb spec
// * synchrnous standby reported from masterdb status
func WaitClusterDataSynchronousStandbys(synchronousStandbys []string, e *store.StoreManager, timeout time.Duration) error {
sort.Sort(sort.StringSlice(synchronousStandbys))
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
if cd.Cluster.Status.Phase == cluster.ClusterPhaseNormal && cd.Cluster.Status.Master != "" {
masterDB := cd.DBs[cd.Cluster.Status.Master]
// get keepers for db spec synchronousStandbys
keepersUIDs := []string{}
for _, dbUID := range masterDB.Spec.SynchronousStandbys {
db, ok := cd.DBs[dbUID]
if ok {
keepersUIDs = append(keepersUIDs, db.Spec.KeeperUID)
}
}
sort.Sort(sort.StringSlice(keepersUIDs))
if !reflect.DeepEqual(synchronousStandbys, keepersUIDs) {
goto end
}
// get keepers for db status synchronousStandbys
keepersUIDs = []string{}
for _, dbUID := range masterDB.Status.SynchronousStandbys {
db, ok := cd.DBs[dbUID]
if ok {
keepersUIDs = append(keepersUIDs, db.Spec.KeeperUID)
}
}
sort.Sort(sort.StringSlice(keepersUIDs))
if !reflect.DeepEqual(synchronousStandbys, keepersUIDs) {
goto end
}
return nil
}
end:
time.Sleep(sleepInterval)
}
return fmt.Errorf("timeout")
}
func WaitClusterDataKeeperInitialized(keeperUID string, e *store.StoreManager, timeout time.Duration) error {
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
// Check for db on keeper to be initialized
for _, db := range cd.DBs {
if db.Spec.KeeperUID == keeperUID {
if db.Status.CurrentGeneration >= cluster.InitialGeneration {
return nil
}
}
}
end:
time.Sleep(sleepInterval)
}
return fmt.Errorf("timeout")
}
func WaitStandbyKeeper(e *store.StoreManager, keeperUID string, timeout time.Duration) error {
start := time.Now()
for time.Now().Add(-timeout).Before(start) {
cd, _, err := e.GetClusterData()
if err != nil || cd == nil {
goto end
}
for _, db := range cd.DBs {
if db.UID == cd.Cluster.Status.Master {
continue
}
if db.Spec.KeeperUID == keeperUID && db.Spec.Role == common.RoleStandby {
return nil
}
}
end:
time.Sleep(sleepInterval)
}
return fmt.Errorf("timeout")
}
