func setupServers(t *testing.T, clusterName, dir string, numKeepers, numSentinels uint8, syncRepl bool, usePgrewind bool) (testKeepers, testSentinels, *TestStore) { initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, SynchronousReplication: cluster.BoolP(syncRepl), UsePgrewind: cluster.BoolP(usePgrewind), PGParameters: make(cluster.PGParameters), } return setupServersCustom(t, clusterName, dir, numKeepers, numSentinels, initialClusterSpec) }
func TestInitialClusterSpec(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) defer tstore.Stop() clusterName := uuid.NewV4().String() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, SynchronousReplication: cluster.BoolP(true), } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer ts.Stop() if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil { t.Fatal("expected cluster in initializing phase") } cd, _, err := sm.GetClusterData() if err != nil { t.Fatalf("unexpected err: %v", err) } if !*cd.Cluster.Spec.SynchronousReplication { t.Fatal("expected cluster spec with SynchronousReplication enabled") } }
func TestInit(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) defer tstore.Stop() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) clusterName := uuid.NewV4().String() initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer ts.Stop() tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tk.Stop() if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } t.Logf("database is up") }
func TestServerParameters(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore, err := NewTestStore(t, dir) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tstore.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tstore.WaitUp(10 * time.Second); err != nil { t.Fatalf("error waiting on store up: %v", err) } storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) defer tstore.Stop() clusterName := uuid.NewV4().String() storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "update", "--patch", `{ "pgParameters" : { "unexistent_parameter": "value" } }`) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.cmd.ExpectTimeout("postgres parameters changed, reloading postgres instance", 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } // On the next keeper check they shouldn't be changed if err := tk.cmd.ExpectTimeout("postgres parameters not changed", 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } tk.Stop() // Start tk again, postgres should fail to start due to bad parameter if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tk.Stop() if err := tk.cmd.ExpectTimeout("failed to start postgres", 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } // Fix wrong parameters err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "update", "--patch", `{ "pgParameters" : null }`) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(30 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } }
func TestLoweredMaxStandbysPerSender(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "stolon") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) clusterName := uuid.NewV4().String() initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, MaxStandbysPerSender: cluster.Uint16P(2), PGParameters: make(cluster.PGParameters), } // Create 3 keepers tks, tss, tstore := setupServersCustom(t, clusterName, dir, 3, 1, initialClusterSpec) defer shutdown(tks, tss, tstore) storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) // Wait for clusterView containing a master masterUID, err := WaitClusterDataWithMaster(sm, 30*time.Second) if err != nil { t.Fatal("expected a master in cluster view") } master := tks[masterUID] waitKeeperReady(t, sm, master) if err := populate(t, master); err != nil { t.Fatalf("unexpected err: %v", err) } if err := write(t, master, 1, 1); err != nil { t.Fatalf("unexpected err: %v", err) } c, err := getLines(t, master) if err != nil { t.Fatalf("unexpected err: %v", err) } if c != 1 { t.Fatalf("wrong number of lines, want: %d, got: %d", 1, c) } if err := WaitNumDBs(sm, 3, 30*time.Second); err != nil { t.Fatalf("expected 3 DBs in cluster data: %v", err) } // Set MaxStandbysPerSender to 1 err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "update", "--patch", `{ "maxStandbysPerSender" : 1 }`) if err != nil { t.Fatalf("unexpected err: %v", err) } // Wait for only 1 standby if err := WaitNumDBs(sm, 2, 30*time.Second); err != nil { t.Fatalf("expected 2 DBs in cluster data: %v", err) } }
func TestFailedStandby(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "stolon") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) clusterName := uuid.NewV4().String() initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, MaxStandbysPerSender: cluster.Uint16P(1), PGParameters: make(cluster.PGParameters), } // Create 3 keepers tks, tss, tstore := setupServersCustom(t, clusterName, dir, 3, 1, initialClusterSpec) defer shutdown(tks, tss, tstore) storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) // Wait for clusterView containing a master masterUID, err := WaitClusterDataWithMaster(sm, 30*time.Second) if err != nil { t.Fatal("expected a master in cluster view") } master := tks[masterUID] waitKeeperReady(t, sm, master) if err := populate(t, master); err != nil { t.Fatalf("unexpected err: %v", err) } if err := write(t, master, 1, 1); err != nil { t.Fatalf("unexpected err: %v", err) } c, err := getLines(t, master) if err != nil { t.Fatalf("unexpected err: %v", err) } if c != 1 { t.Fatalf("wrong number of lines, want: %d, got: %d", 1, c) } if err := WaitNumDBs(sm, 2, 30*time.Second); err != nil { t.Fatalf("expected 2 DBs in cluster data: %v", err) } cd, _, err := sm.GetClusterData() if err != nil { t.Fatalf("unexpected err: %v", err) } // Get current standby var standby *TestKeeper for _, db := range cd.DBs { if db.UID == cd.Cluster.Status.Master { continue } standby = tks[db.Spec.KeeperUID] } if err := waitLines(t, standby, 1, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } // Stop current standby. The other keeper should be choosed as new standby t.Logf("Stopping current standby keeper: %s", standby.uid) standby.Stop() // Wait for other keeper to have a standby db assigned var newStandby *TestKeeper for _, tk := range tks { if tk.uid != master.uid && tk.uid != standby.uid { newStandby = tk } } if err := WaitStandbyKeeper(sm, newStandby.uid, 20*time.Second); err != nil { t.Fatalf("expected keeper %s to have a standby db assigned: %v", newStandby.uid, err) } // Wait for new standby declared as good and remove of old standby if err := WaitNumDBs(sm, 2, 30*time.Second); err != nil { t.Fatalf("expected 2 DBs in cluster data: %v", err) } }
func TestInitWithMultipleKeepers(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "stolon") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) clusterName := uuid.NewV4().String() storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), FailInterval: &cluster.Duration{Duration: 10 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } tks := testKeepers{} tss := testSentinels{} // Start 3 keepers for i := uint8(0); i < 3; i++ { tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tks[tk.uid] = tk } // Start 2 sentinels for i := uint8(0); i < 2; i++ { ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tss[ts.uid] = ts } defer shutdown(tks, tss, tstore) // Wait for clusterView containing a master masterUID, err := WaitClusterDataWithMaster(sm, 30*time.Second) if err != nil { t.Fatal("expected a master in cluster view") } waitKeeperReady(t, sm, tks[masterUID]) }
func TestProxyListening(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) clusterName := uuid.NewV4().String() tstore, err := NewTestStore(t, dir) if err != nil { t.Fatalf("unexpected err: %v", err) } storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) tp, err := NewTestProxy(t, dir, clusterName, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tp.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tp.Stop() t.Logf("test proxy start with store down. Should not listen") // tp should not listen because it cannot talk with store if err := tp.WaitNotListening(10 * time.Second); err != nil { t.Fatalf("expecting tp not listening due to failed store communication, but it's listening.") } tp.Stop() if err := tstore.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tstore.WaitUp(10 * time.Second); err != nil { t.Fatalf("error waiting on store up: %v", err) } defer func() { if tstore.cmd != nil { tstore.Stop() } }() storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) cd := &cluster.ClusterData{ FormatVersion: cluster.CurrentCDFormatVersion, Cluster: &cluster.Cluster{ UID: "01", Generation: 1, Spec: &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), FailInterval: &cluster.Duration{Duration: 10 * time.Second}, }, Status: cluster.ClusterStatus{ CurrentGeneration: 1, Phase: cluster.ClusterPhaseNormal, Master: "01", }, }, Keepers: cluster.Keepers{ "01": &cluster.Keeper{ UID: "01", Spec: &cluster.KeeperSpec{}, Status: cluster.KeeperStatus{ Healthy: true, }, }, }, DBs: cluster.DBs{ "01": &cluster.DB{ UID: "01", Generation: 1, ChangeTime: time.Time{}, Spec: &cluster.DBSpec{ KeeperUID: "01", Role: common.RoleMaster, Followers: []string{"02"}, }, Status: cluster.DBStatus{ Healthy: false, CurrentGeneration: 1, }, }, }, Proxy: &cluster.Proxy{ Spec: cluster.ProxySpec{ MasterDBUID: "01", }, }, } pair, err := sm.AtomicPutClusterData(cd, nil) if err != nil { t.Fatalf("unexpected err: %v", err) } // test proxy start with the store up t.Logf("test proxy start with the store up. Should listen") if err := tp.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } // tp should listen if err := tp.WaitListening(10 * time.Second); err != nil { t.Fatalf("expecting tp listening, but it's not listening.") } t.Logf("test proxy error communicating with store. Should stop listening") // Stop store tstore.Stop() if err := tstore.WaitDown(10 * time.Second); err != nil { t.Fatalf("error waiting on store down: %v", err) } // tp should not listen because it cannot talk with the store if err := tp.WaitNotListening(10 * time.Second); err != nil { t.Fatalf("expecting tp not listening due to failed store communication, but it's listening.") } t.Logf("test proxy communication with store restored. Should start listening") // Start store if err := tstore.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tstore.WaitUp(10 * time.Second); err != nil { t.Fatalf("error waiting on store up: %v", err) } // tp should listen if err := tp.WaitListening(10 * time.Second); err != nil { t.Fatalf("expecting tp listening, but it's not listening.") } t.Logf("test proxyConf removed. Should continue listening") // remove proxyConf cd.Proxy.Spec.MasterDBUID = "" pair, err = sm.AtomicPutClusterData(cd, pair) if err != nil { t.Fatalf("unexpected err: %v", err) } // tp should listen if err := tp.WaitListening(10 * time.Second); err != nil { t.Fatalf("expecting tp listening, but it's not listening.") } t.Logf("test proxyConf restored. Should continue listening") // Set proxyConf again cd.Proxy.Spec.MasterDBUID = "01" pair, err = sm.AtomicPutClusterData(cd, pair) if err != nil { t.Fatalf("unexpected err: %v", err) } // tp should listen if err := tp.WaitListening(10 * time.Second); err != nil { t.Fatalf("expecting tp listening, but it's not listening.") } t.Logf("test clusterView removed. Should continue listening") // remove whole clusterview _, err = sm.AtomicPutClusterData(nil, pair) if err != nil { t.Fatalf("unexpected err: %v", err) } // tp should listen if err := tp.WaitListening(10 * time.Second); err != nil { t.Fatalf("expecting tp listening, but it's not listening.") } }
func testInitNew(t *testing.T, merge bool) { clusterName := uuid.NewV4().String() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) defer tstore.Stop() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), FailInterval: &cluster.Duration{Duration: 10 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, MergePgParameters: &merge, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } cd, _, err := sm.GetClusterData() // max_connection should be set by initdb _, ok := cd.Cluster.Spec.PGParameters["max_connections"] if merge && !ok { t.Fatalf("expected max_connection set in cluster data pgParameters") } if !merge && ok { t.Fatalf("expected no max_connection set in cluster data pgParameters") } tk.Stop() }
func TestInitUsers(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) defer tstore.Stop() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) // Test pg-repl-username == pg-su-username but password different clusterName := uuid.NewV4().String() tk, err := NewTestKeeper(t, dir, clusterName, "user01", "password01", "user01", "password02", tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.StartExpect(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tk.Stop() if err := tk.cmd.Expect("provided superuser name and replication user name are the same but provided passwords are different"); err != nil { t.Fatalf("expecting keeper reporting provided superuser name and replication user name are the same but provided passwords are different") } // Test pg-repl-username == pg-su-username clusterName = uuid.NewV4().String() storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer ts.Stop() if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 30*time.Second); err != nil { t.Fatal("expected cluster in initializing phase") } tk2, err := NewTestKeeper(t, dir, clusterName, "user01", "password", "user01", "password", tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk2.StartExpect(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tk2.Stop() if err := tk2.cmd.ExpectTimeout("replication role added to superuser", 60*time.Second); err != nil { t.Fatalf("expecting keeper reporting replication role added to superuser") } // Test pg-repl-username != pg-su-username and pg-su-password defined clusterName = uuid.NewV4().String() storePath = filepath.Join(common.StoreBasePath, clusterName) sm = store.NewStoreManager(tstore.store, storePath) ts2, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts2.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer ts2.Stop() if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil { t.Fatal("expected cluster in initializing phase") } tk3, err := NewTestKeeper(t, dir, clusterName, "user01", "password", "user02", "password", tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk3.StartExpect(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tk3.Stop() if err := tk3.cmd.ExpectTimeout("superuser password set", 60*time.Second); err != nil { t.Fatalf("expecting keeper reporting superuser password set") } if err := tk3.cmd.ExpectTimeout("replication role created role=user02", 60*time.Second); err != nil { t.Fatalf("expecting keeper reporting replication role user02 created") } }
func testInitExisting(t *testing.T, merge bool) { clusterName := uuid.NewV4().String() dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) tstore := setupStore(t, dir) defer tstore.Stop() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, PGParameters: cluster.PGParameters{ "archive_mode": "on", }, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := populate(t, tk); err != nil { t.Fatalf("unexpected err: %v", err) } if err := write(t, tk, 1, 1); err != nil { t.Fatalf("unexpected err: %v", err) } // Now initialize a new cluster with the existing keeper initialClusterSpec = &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeExisting), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, MergePgParameters: &merge, ExistingConfig: &cluster.ExistingConfig{ KeeperUID: tk.uid, }, } initialClusterSpecFile, err = writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } t.Logf("reinitializing cluster") // Initialize cluster with new spec err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "init", "-y", "-f", initialClusterSpecFile) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } c, err := getLines(t, tk) if err != nil { t.Fatalf("unexpected err: %v", err) } if c != 1 { t.Fatalf("wrong number of lines, want: %d, got: %d", 1, c) } pgParameters, err := tk.GetPGParameters() if err != nil { t.Fatalf("unexpected err: %v", err) } v, ok := pgParameters["archive_mode"] if merge && v != "on" { t.Fatalf("expected archive_mode == on got %q", v) } if !merge && ok { t.Fatalf("expected archive_mode empty") } cd, _, err := sm.GetClusterData() // max_connection should be set by initdb v, ok = cd.Cluster.Spec.PGParameters["archive_mode"] if merge && v != "on" { t.Fatalf("expected archive_mode == on got %q", v) } if !merge && ok { t.Fatalf("expected archive_mode empty") } tk.Stop() }
func TestPITR(t *testing.T) { t.Parallel() dir, err := ioutil.TempDir("", "stolon") if err != nil { t.Fatalf("unexpected err: %v", err) } defer os.RemoveAll(dir) baseBackupDir, err := ioutil.TempDir(dir, "basebackup") if err != nil { t.Fatalf("unexpected err: %v", err) } archiveBackupDir, err := ioutil.TempDir(dir, "archivebackup") if err != nil { t.Fatalf("unexpected err: %v", err) } tstore := setupStore(t, dir) defer tstore.Stop() storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port) clusterName := uuid.NewV4().String() storePath := filepath.Join(common.StoreBasePath, clusterName) sm := store.NewStoreManager(tstore.store, storePath) initialClusterSpec := &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, PGParameters: cluster.PGParameters{ "archive_mode": "on", "archive_command": fmt.Sprintf("cp %%p %s/%%f", archiveBackupDir), }, } initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer tk.Stop() ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } // Wait for clusterView containing a master _, err = WaitClusterDataWithMaster(sm, 30*time.Second) if err != nil { t.Fatal("expected a master in cluster view") } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitRole(common.RoleMaster, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := populate(t, tk); err != nil { t.Fatalf("unexpected err: %v", err) } if err := write(t, tk, 2, 2); err != nil { t.Fatalf("unexpected err: %v", err) } // ioutil.Tempfile already creates files with 0600 permissions pgpass, err := ioutil.TempFile("", "pgpass") if err != nil { t.Fatalf("unexpected err: %v", err) } pgpass.WriteString(fmt.Sprintf("%s:%s:*:%s:%s\n", tk.pgListenAddress, tk.pgPort, tk.pgReplUsername, tk.pgReplPassword)) // Don't save the wal during the basebackup (-x). This to test that archive_command and restore command correctly work. cmd := exec.Command("pg_basebackup", "-F", "tar", "-D", baseBackupDir, "-h", tk.pgListenAddress, "-p", tk.pgPort, "-U", tk.pgReplUsername) cmd.Env = append(cmd.Env, fmt.Sprintf("PGPASSFILE=%s", pgpass.Name())) t.Logf("execing cmd: %s", cmd) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("error: %v, output: %s", err, string(out)) } // Switch wal so they will be archived if _, err := tk.db.Exec("select pg_switch_xlog()"); err != nil { t.Fatalf("unexpected err: %v", err) } ts.Stop() // Delete the current cluster data if err := tstore.store.Delete(filepath.Join(storePath, "clusterdata")); err != nil { t.Fatalf("unexpected err: %v", err) } // Delete sentinel leader key to just speedup new election if err := tstore.store.Delete(filepath.Join(storePath, common.SentinelLeaderKey)); err != nil { t.Fatalf("unexpected err: %v", err) } // Now initialize a new cluster with the existing keeper initialClusterSpec = &cluster.ClusterSpec{ InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModePITR), SleepInterval: &cluster.Duration{Duration: 2 * time.Second}, FailInterval: &cluster.Duration{Duration: 5 * time.Second}, ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second}, PITRConfig: &cluster.PITRConfig{ DataRestoreCommand: fmt.Sprintf("tar xvf %s/base.tar -C %%d", baseBackupDir), ArchiveRecoverySettings: &cluster.ArchiveRecoverySettings{ RestoreCommand: fmt.Sprintf("cp %s/%%f %%p", archiveBackupDir), }, }, } initialClusterSpecFile, err = writeClusterSpec(dir, initialClusterSpec) if err != nil { t.Fatalf("unexpected err: %v", err) } ts, err = NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile)) if err != nil { t.Fatalf("unexpected err: %v", err) } if err := ts.Start(); err != nil { t.Fatalf("unexpected err: %v", err) } defer ts.Stop() if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } _, err = WaitClusterDataWithMaster(sm, 30*time.Second) if err != nil { t.Fatal("expected a master in cluster view") } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitRole(common.RoleMaster, 30*time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } if err := tk.WaitDBUp(60 * time.Second); err != nil { t.Fatalf("unexpected err: %v", err) } c, err := getLines(t, tk) if err != nil { t.Fatalf("unexpected err: %v", err) } if c != 1 { t.Fatalf("wrong number of lines, want: %d, got: %d", 2, c) } }