func setupServers(t *testing.T, clusterName, dir string, numKeepers, numSentinels uint8, syncRepl bool, usePgrewind bool) (testKeepers, testSentinels, *TestStore) { initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, SynchronousReplication: cluster.BoolP(syncRepl), UsePgrewind: cluster.BoolP(usePgrewind), PGParameters: make(cluster.PGParameters), } return setupServersCustom(t, clusterName, dir, numKeepers, numSentinels, initialClusterSpec) }
func TestInitialClusterSpec(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) defer tstore.Stop() clusterName := uuid.NewV4().String() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, SynchronousReplication: cluster.BoolP(true), } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer ts.Stop() if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil { t.Fatal("expected cluster in initializing phase") } cd, _, err := sm.GetClusterData() if err != nil { t.Fatalf("unexpected err: %v", err) } if !*cd.Cluster.Spec.SynchronousReplication { t.Fatal("expected cluster spec with SynchronousReplication enabled") } }
func TestUpdateClusterView(t *testing.T) { tests := []struct { cv *cluster.ClusterView keepersState cluster.KeepersState outCV *cluster.ClusterView err error }{ { cv: cluster.NewClusterView(), keepersState: nil, outCV: cluster.NewClusterView(), err: fmt.Errorf("cluster view at version 0 without a defined master. This shouldn't happen!"), }, { cv: &cluster.ClusterView{ Version: 1, KeepersRole: cluster.KeepersRole{}, }, keepersState: nil, outCV: &cluster.ClusterView{ Version: 1, KeepersRole: cluster.KeepersRole{}, }, err: fmt.Errorf("cannot choose initial master, no keepers registered"), }, // cluster initialization, one keeper { cv: &cluster.ClusterView{ Version: 1, KeepersRole: cluster.KeepersRole{}, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{PGState: &cluster.PostgresState{Initialized: true}}, }, outCV: &cluster.ClusterView{ Version: 2, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, }, }, }, // cluster initialization, too many keepers { cv: &cluster.ClusterView{ Version: 1, KeepersRole: cluster.KeepersRole{}, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{}, "02": &cluster.KeeperState{}, }, outCV: &cluster.ClusterView{ Version: 1, KeepersRole: cluster.KeepersRole{}, }, err: fmt.Errorf("cannot choose initial master, more than 1 keeper registered"), }, // cluster initialization, more then one keeper but InitWithMultipleKeepers == true { cv: &cluster.ClusterView{ Version: 1, KeepersRole: cluster.KeepersRole{}, Config: &cluster.NilConfig{InitWithMultipleKeepers: cluster.BoolP(true)}, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{PGState: &cluster.PostgresState{Initialized: true}}, "02": &cluster.KeeperState{PGState: &cluster.PostgresState{Initialized: true}}, }, outCV: &cluster.ClusterView{ Version: 2, KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: ""}, }, Config: &cluster.NilConfig{InitWithMultipleKeepers: cluster.BoolP(true)}, }, }, // One master and one standby, both healthy: no change from previous cv { cv: &cluster.ClusterView{ Version: 1, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: "01"}, }, ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"}, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "01", PGPort: "01", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, "02": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "02", PGPort: "02", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, }, outCV: &cluster.ClusterView{ Version: 1, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: "01"}, }, ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"}, }, }, // One master and one standby, master not healthy: standby elected as new master { cv: &cluster.ClusterView{ Version: 1, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: "01"}, }, ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"}, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "01", PGPort: "01", ErrorStartTime: time.Unix(0, 0), Healthy: false, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, "02": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "02", PGPort: "02", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, }, outCV: &cluster.ClusterView{ Version: 2, Master: "02", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: ""}, }, ProxyConf: nil, }, }, // From the previous test, new master (02) converged. Old master setup to follow new master (02). { cv: &cluster.ClusterView{ Version: 2, Master: "02", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: ""}, }, ProxyConf: nil, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "01", PGPort: "01", ErrorStartTime: time.Unix(0, 0), Healthy: false, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, "02": &cluster.KeeperState{ ClusterViewVersion: 2, PGListenAddress: "02", PGPort: "02", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, }, outCV: &cluster.ClusterView{ Version: 3, Master: "02", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: "02"}, "02": &cluster.KeeperRole{ID: "02", Follow: ""}, }, ProxyConf: &cluster.ProxyConf{Host: "02", Port: "02"}, }, }, // One master and one standby, master not healthy, standby with old // clusterview: no standby elected as new master. { cv: &cluster.ClusterView{ Version: 2, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: "01"}, }, ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"}, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{ ClusterViewVersion: 2, PGListenAddress: "01", PGPort: "01", ErrorStartTime: time.Unix(0, 0), Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, "02": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "02", PGPort: "02", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, }, outCV: &cluster.ClusterView{ Version: 2, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: "01"}, }, ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"}, }, }, // One master and one standby, master not converged to current // cv: standby elected as new master. { cv: &cluster.ClusterView{ Version: 2, Master: "01", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: "01"}, }, ProxyConf: nil, }, keepersState: cluster.KeepersState{ "01": &cluster.KeeperState{ ClusterViewVersion: 1, PGListenAddress: "01", PGPort: "01", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, "02": &cluster.KeeperState{ ClusterViewVersion: 2, PGListenAddress: "02", PGPort: "02", ErrorStartTime: time.Time{}, Healthy: true, PGState: &cluster.PostgresState{ TimelineID: 0, }, }, }, outCV: &cluster.ClusterView{ Version: 3, Master: "02", KeepersRole: cluster.KeepersRole{ "01": &cluster.KeeperRole{ID: "01", Follow: ""}, "02": &cluster.KeeperRole{ID: "02", Follow: ""}, }, ProxyConf: nil, }, }, } for i, tt := range tests { var s *Sentinel if tt.cv.Config == nil { s = &Sentinel{id: "id", clusterConfig: cluster.NewDefaultConfig()} } else { s = &Sentinel{id: "id", clusterConfig: tt.cv.Config.ToConfig()} } outCV, err := s.updateClusterView(tt.cv, tt.keepersState) t.Logf("test #%d", i) t.Logf(spew.Sprintf("outCV: %#v", outCV)) if tt.err != nil { if err == nil { t.Errorf("got no error, wanted error: %v", tt.err) } else if tt.err.Error() != err.Error() { t.Errorf("got error: %v, wanted error: %v", err, tt.err) } } else { if err != nil { t.Errorf("unexpected error: %v", err) } if !outCV.Equals(tt.outCV) { t.Errorf(spew.Sprintf("#%d: wrong outCV: got: %#v, want: %#v", i, outCV, tt.outCV)) } } } }
func setupServers(t *testing.T, dir string, numKeepers, numSentinels uint8, syncRepl bool) ([]*TestKeeper, []*TestSentinel, *TestEtcd) { te, err := NewTestEtcd(dir) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := te.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := te.WaitUp(10 * time.Second); err != nil { t.Fatalf("error waiting on etcd up: %v", err) } etcdEndpoints := fmt.Sprintf("http://%s:%s", te.listenAddress, te.port) clusterName := uuid.NewV4().String() etcdPath := filepath.Join(common.EtcdBasePath, clusterName) e, err := etcdm.NewEtcdManager(etcdEndpoints, etcdPath, common.DefaultEtcdRequestTimeout) if err != nil { t.Fatalf("cannot create etcd manager: %v", err) } // TODO(sgotti) change this to a call to the sentinel to change the // cluster config (when the sentinel's code is done) e.SetClusterData(cluster.KeepersState{}, &cluster.ClusterView{ Version: 1, Config: &cluster.NilConfig{ SleepInterval: cluster.DurationP(5 * time.Second), KeeperFailInterval: cluster.DurationP(10 * time.Second), SynchronousReplication: cluster.BoolP(syncRepl), }, }, 0) tks := []*TestKeeper{} tss := []*TestSentinel{} tk, err := NewTestKeeper(dir, clusterName, etcdEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } tks = append(tks, tk) fmt.Printf("tk: %v\n", tk) // Start sentinels for i := uint8(0); i < numSentinels; i++ { ts, err := NewTestSentinel(dir, clusterName, etcdEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tss = append(tss, ts) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitRole(common.MasterRole, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } // Wait for clusterView containing tk as master if err := WaitClusterViewMaster(tk.id, e, 30*time.Second); err != nil { t.Fatalf("expected master %q in cluster view", tk.id) } // Start standbys for i := uint8(1); i < numKeepers; i++ { tk, err := NewTestKeeper(dir, clusterName, etcdEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitRole(common.StandbyRole, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } tks = append(tks, tk) } return tks, tss, te }
func TestInitWithMultipleKeepers(t *testing.T) { dir, err := ioutil.TempDir("", "stolon") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) clusterName := uuid.NewV4().String() storePath := filepath.Join(common.StoreBasePath, clusterName) kvstore, err := store.NewStore(tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("cannot create store: %v", err) } e := store.NewStoreManager(kvstore, storePath) // TODO(sgotti) change this to a call to the sentinel to change the // cluster config (when the sentinel's code is done) e.SetClusterData(cluster.KeepersState{}, &cluster.ClusterView{ Version: 1, Config: &cluster.NilConfig{ SleepInterval: &cluster.Duration{5 * time.Second}, KeeperFailInterval: &cluster.Duration{10 * time.Second}, InitWithMultipleKeepers: cluster.BoolP(true), }, }, nil) tks := []*TestKeeper{} tss := []*TestSentinel{} // Start 3 keepers for i := uint8(0); i < 3; i++ { tk, err := NewTestKeeper(dir, clusterName, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tks = append(tks, tk) if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } } // Start 2 sentinels for i := uint8(0); i < 2; i++ { ts, err := NewTestSentinel(dir, clusterName, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tss = append(tss, ts) } defer shutdown(tks, tss, tstore) // Wait for clusterView containing a master if err := WaitClusterViewWithMaster(e, 30*time.Second); err != nil { t.Fatal("expected a master in cluster view") } }
func setupServers(t *testing.T, dir string, numKeepers, numSentinels uint8, syncRepl bool) ([]*TestKeeper, []*TestSentinel, *TestStore) { tstore := setupStore(t, dir) storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) clusterName := uuid.NewV4().String() storePath := filepath.Join(common.StoreBasePath, clusterName) kvstore, err := store.NewStore(tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("cannot create store: %v", err) } e := store.NewStoreManager(kvstore, storePath) // TODO(sgotti) change this to a call to the sentinel to change the // cluster config (when the sentinel's code is done) e.SetClusterData(cluster.KeepersState{}, &cluster.ClusterView{ Version: 1, Config: &cluster.NilConfig{ SleepInterval: &cluster.Duration{5 * time.Second}, KeeperFailInterval: &cluster.Duration{10 * time.Second}, SynchronousReplication: cluster.BoolP(syncRepl), }, }, nil) tks := []*TestKeeper{} tss := []*TestSentinel{} tk, err := NewTestKeeper(dir, clusterName, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } tks = append(tks, tk) fmt.Printf("tk: %v\n", tk) // Start sentinels for i := uint8(0); i < numSentinels; i++ { ts, err := NewTestSentinel(dir, clusterName, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tss = append(tss, ts) } // Start first keeper if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitRole(common.MasterRole, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } // Wait for clusterView containing tk as master if err := WaitClusterViewMaster(tk.id, e, 30*time.Second); err != nil { t.Fatalf("expected master %q in cluster view", tk.id) } // Start other keepers for i := uint8(1); i < numKeepers; i++ { tk, err := NewTestKeeper(dir, clusterName, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } // Wait for clusterView containing tk as standby if err := tk.WaitRole(common.StandbyRole, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } tks = append(tks, tk) } return tks, tss, tstore }