func setupServers(t *testing.T, clusterName, dir string, numKeepers, numSentinels uint8, syncRepl bool, usePgrewind bool) (testKeepers, testSentinels, *TestStore) {
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
SynchronousReplication: cluster.BoolP(syncRepl),
UsePgrewind: cluster.BoolP(usePgrewind),
PGParameters: make(cluster.PGParameters),
}
return setupServersCustom(t, clusterName, dir, numKeepers, numSentinels, initialClusterSpec)
}
func TestInitialClusterSpec(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
defer tstore.Stop()
clusterName := uuid.NewV4().String()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
SynchronousReplication: cluster.BoolP(true),
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer ts.Stop()
if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil {
t.Fatal("expected cluster in initializing phase")
}
cd, _, err := sm.GetClusterData()
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if !*cd.Cluster.Spec.SynchronousReplication {
t.Fatal("expected cluster spec with SynchronousReplication enabled")
}
}
func TestInit(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
defer tstore.Stop()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
clusterName := uuid.NewV4().String()
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer ts.Stop()
tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tk.Stop()
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("database is up")
}
func TestServerParameters(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore, err := NewTestStore(t, dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tstore.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tstore.WaitUp(10 * time.Second); err != nil {
t.Fatalf("error waiting on store up: %v", err)
}
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
defer tstore.Stop()
clusterName := uuid.NewV4().String()
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "update", "--patch", `{ "pgParameters" : { "unexistent_parameter": "value" } }`)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.cmd.ExpectTimeout("postgres parameters changed, reloading postgres instance", 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// On the next keeper check they shouldn't be changed
if err := tk.cmd.ExpectTimeout("postgres parameters not changed", 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tk.Stop()
// Start tk again, postgres should fail to start due to bad parameter
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tk.Stop()
if err := tk.cmd.ExpectTimeout("failed to start postgres", 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Fix wrong parameters
err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "update", "--patch", `{ "pgParameters" : null }`)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(30 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
}
func TestLoweredMaxStandbysPerSender(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "stolon")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
clusterName := uuid.NewV4().String()
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
MaxStandbysPerSender: cluster.Uint16P(2),
PGParameters: make(cluster.PGParameters),
}
// Create 3 keepers
tks, tss, tstore := setupServersCustom(t, clusterName, dir, 3, 1, initialClusterSpec)
defer shutdown(tks, tss, tstore)
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
// Wait for clusterView containing a master
masterUID, err := WaitClusterDataWithMaster(sm, 30*time.Second)
if err != nil {
t.Fatal("expected a master in cluster view")
}
master := tks[masterUID]
waitKeeperReady(t, sm, master)
if err := populate(t, master); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := write(t, master, 1, 1); err != nil {
t.Fatalf("unexpected err: %v", err)
}
c, err := getLines(t, master)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if c != 1 {
t.Fatalf("wrong number of lines, want: %d, got: %d", 1, c)
}
if err := WaitNumDBs(sm, 3, 30*time.Second); err != nil {
t.Fatalf("expected 3 DBs in cluster data: %v", err)
}
// Set MaxStandbysPerSender to 1
err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "update", "--patch", `{ "maxStandbysPerSender" : 1 }`)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Wait for only 1 standby
if err := WaitNumDBs(sm, 2, 30*time.Second); err != nil {
t.Fatalf("expected 2 DBs in cluster data: %v", err)
}
}
func TestFailedStandby(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "stolon")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
clusterName := uuid.NewV4().String()
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
MaxStandbysPerSender: cluster.Uint16P(1),
PGParameters: make(cluster.PGParameters),
}
// Create 3 keepers
tks, tss, tstore := setupServersCustom(t, clusterName, dir, 3, 1, initialClusterSpec)
defer shutdown(tks, tss, tstore)
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
// Wait for clusterView containing a master
masterUID, err := WaitClusterDataWithMaster(sm, 30*time.Second)
if err != nil {
t.Fatal("expected a master in cluster view")
}
master := tks[masterUID]
waitKeeperReady(t, sm, master)
if err := populate(t, master); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := write(t, master, 1, 1); err != nil {
t.Fatalf("unexpected err: %v", err)
}
c, err := getLines(t, master)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if c != 1 {
t.Fatalf("wrong number of lines, want: %d, got: %d", 1, c)
}
if err := WaitNumDBs(sm, 2, 30*time.Second); err != nil {
t.Fatalf("expected 2 DBs in cluster data: %v", err)
}
cd, _, err := sm.GetClusterData()
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Get current standby
var standby *TestKeeper
for _, db := range cd.DBs {
if db.UID == cd.Cluster.Status.Master {
continue
}
standby = tks[db.Spec.KeeperUID]
}
if err := waitLines(t, standby, 1, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Stop current standby. The other keeper should be choosed as new standby
t.Logf("Stopping current standby keeper: %s", standby.uid)
standby.Stop()
// Wait for other keeper to have a standby db assigned
var newStandby *TestKeeper
for _, tk := range tks {
if tk.uid != master.uid && tk.uid != standby.uid {
newStandby = tk
}
}
if err := WaitStandbyKeeper(sm, newStandby.uid, 20*time.Second); err != nil {
t.Fatalf("expected keeper %s to have a standby db assigned: %v", newStandby.uid, err)
}
// Wait for new standby declared as good and remove of old standby
if err := WaitNumDBs(sm, 2, 30*time.Second); err != nil {
t.Fatalf("expected 2 DBs in cluster data: %v", err)
}
}
func TestInitWithMultipleKeepers(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "stolon")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
clusterName := uuid.NewV4().String()
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
FailInterval: &cluster.Duration{Duration: 10 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks := testKeepers{}
tss := testSentinels{}
// Start 3 keepers
for i := uint8(0); i < 3; i++ {
tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks[tk.uid] = tk
}
// Start 2 sentinels
for i := uint8(0); i < 2; i++ {
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tss[ts.uid] = ts
}
defer shutdown(tks, tss, tstore)
// Wait for clusterView containing a master
masterUID, err := WaitClusterDataWithMaster(sm, 30*time.Second)
if err != nil {
t.Fatal("expected a master in cluster view")
}
waitKeeperReady(t, sm, tks[masterUID])
}
func TestProxyListening(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
clusterName := uuid.NewV4().String()
tstore, err := NewTestStore(t, dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
tp, err := NewTestProxy(t, dir, clusterName, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tp.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tp.Stop()
t.Logf("test proxy start with store down. Should not listen")
// tp should not listen because it cannot talk with store
if err := tp.WaitNotListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp not listening due to failed store communication, but it's listening.")
}
tp.Stop()
if err := tstore.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tstore.WaitUp(10 * time.Second); err != nil {
t.Fatalf("error waiting on store up: %v", err)
}
defer func() {
if tstore.cmd != nil {
tstore.Stop()
}
}()
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
cd := &cluster.ClusterData{
FormatVersion: cluster.CurrentCDFormatVersion,
Cluster: &cluster.Cluster{
UID: "01",
Generation: 1,
Spec: &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
FailInterval: &cluster.Duration{Duration: 10 * time.Second},
},
Status: cluster.ClusterStatus{
CurrentGeneration: 1,
Phase: cluster.ClusterPhaseNormal,
Master: "01",
},
},
Keepers: cluster.Keepers{
"01": &cluster.Keeper{
UID: "01",
Spec: &cluster.KeeperSpec{},
Status: cluster.KeeperStatus{
Healthy: true,
},
},
},
DBs: cluster.DBs{
"01": &cluster.DB{
UID: "01",
Generation: 1,
ChangeTime: time.Time{},
Spec: &cluster.DBSpec{
KeeperUID: "01",
Role: common.RoleMaster,
Followers: []string{"02"},
},
Status: cluster.DBStatus{
Healthy: false,
CurrentGeneration: 1,
},
},
},
Proxy: &cluster.Proxy{
Spec: cluster.ProxySpec{
MasterDBUID: "01",
},
},
}
pair, err := sm.AtomicPutClusterData(cd, nil)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// test proxy start with the store up
t.Logf("test proxy start with the store up. Should listen")
if err := tp.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// tp should listen
if err := tp.WaitListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp listening, but it's not listening.")
}
t.Logf("test proxy error communicating with store. Should stop listening")
// Stop store
tstore.Stop()
if err := tstore.WaitDown(10 * time.Second); err != nil {
t.Fatalf("error waiting on store down: %v", err)
}
// tp should not listen because it cannot talk with the store
if err := tp.WaitNotListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp not listening due to failed store communication, but it's listening.")
}
t.Logf("test proxy communication with store restored. Should start listening")
// Start store
if err := tstore.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tstore.WaitUp(10 * time.Second); err != nil {
t.Fatalf("error waiting on store up: %v", err)
}
// tp should listen
if err := tp.WaitListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp listening, but it's not listening.")
}
t.Logf("test proxyConf removed. Should continue listening")
// remove proxyConf
cd.Proxy.Spec.MasterDBUID = ""
pair, err = sm.AtomicPutClusterData(cd, pair)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// tp should listen
if err := tp.WaitListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp listening, but it's not listening.")
}
t.Logf("test proxyConf restored. Should continue listening")
// Set proxyConf again
cd.Proxy.Spec.MasterDBUID = "01"
pair, err = sm.AtomicPutClusterData(cd, pair)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// tp should listen
if err := tp.WaitListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp listening, but it's not listening.")
}
t.Logf("test clusterView removed. Should continue listening")
// remove whole clusterview
_, err = sm.AtomicPutClusterData(nil, pair)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
// tp should listen
if err := tp.WaitListening(10 * time.Second); err != nil {
t.Fatalf("expecting tp listening, but it's not listening.")
}
}
func testInitNew(t *testing.T, merge bool) {
clusterName := uuid.NewV4().String()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
defer tstore.Stop()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
FailInterval: &cluster.Duration{Duration: 10 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
MergePgParameters: &merge,
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
cd, _, err := sm.GetClusterData()
// max_connection should be set by initdb
_, ok := cd.Cluster.Spec.PGParameters["max_connections"]
if merge && !ok {
t.Fatalf("expected max_connection set in cluster data pgParameters")
}
if !merge && ok {
t.Fatalf("expected no max_connection set in cluster data pgParameters")
}
tk.Stop()
}
func TestInitUsers(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
defer tstore.Stop()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
// Test pg-repl-username == pg-su-username but password different
clusterName := uuid.NewV4().String()
tk, err := NewTestKeeper(t, dir, clusterName, "user01", "password01", "user01", "password02", tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.StartExpect(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tk.Stop()
if err := tk.cmd.Expect("provided superuser name and replication user name are the same but provided passwords are different"); err != nil {
t.Fatalf("expecting keeper reporting provided superuser name and replication user name are the same but provided passwords are different")
}
// Test pg-repl-username == pg-su-username
clusterName = uuid.NewV4().String()
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer ts.Stop()
if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 30*time.Second); err != nil {
t.Fatal("expected cluster in initializing phase")
}
tk2, err := NewTestKeeper(t, dir, clusterName, "user01", "password", "user01", "password", tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk2.StartExpect(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tk2.Stop()
if err := tk2.cmd.ExpectTimeout("replication role added to superuser", 60*time.Second); err != nil {
t.Fatalf("expecting keeper reporting replication role added to superuser")
}
// Test pg-repl-username != pg-su-username and pg-su-password defined
clusterName = uuid.NewV4().String()
storePath = filepath.Join(common.StoreBasePath, clusterName)
sm = store.NewStoreManager(tstore.store, storePath)
ts2, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts2.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer ts2.Stop()
if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil {
t.Fatal("expected cluster in initializing phase")
}
tk3, err := NewTestKeeper(t, dir, clusterName, "user01", "password", "user02", "password", tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk3.StartExpect(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tk3.Stop()
if err := tk3.cmd.ExpectTimeout("superuser password set", 60*time.Second); err != nil {
t.Fatalf("expecting keeper reporting superuser password set")
}
if err := tk3.cmd.ExpectTimeout("replication role created role=user02", 60*time.Second); err != nil {
t.Fatalf("expecting keeper reporting replication role user02 created")
}
}
func testInitExisting(t *testing.T, merge bool) {
clusterName := uuid.NewV4().String()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
defer tstore.Stop()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
PGParameters: cluster.PGParameters{
"archive_mode": "on",
},
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := populate(t, tk); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := write(t, tk, 1, 1); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Now initialize a new cluster with the existing keeper
initialClusterSpec = &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeExisting),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
MergePgParameters: &merge,
ExistingConfig: &cluster.ExistingConfig{
KeeperUID: tk.uid,
},
}
initialClusterSpecFile, err = writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
t.Logf("reinitializing cluster")
// Initialize cluster with new spec
err = StolonCtl(clusterName, tstore.storeBackend, storeEndpoints, "init", "-y", "-f", initialClusterSpecFile)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
c, err := getLines(t, tk)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if c != 1 {
t.Fatalf("wrong number of lines, want: %d, got: %d", 1, c)
}
pgParameters, err := tk.GetPGParameters()
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
v, ok := pgParameters["archive_mode"]
if merge && v != "on" {
t.Fatalf("expected archive_mode == on got %q", v)
}
if !merge && ok {
t.Fatalf("expected archive_mode empty")
}
cd, _, err := sm.GetClusterData()
// max_connection should be set by initdb
v, ok = cd.Cluster.Spec.PGParameters["archive_mode"]
if merge && v != "on" {
t.Fatalf("expected archive_mode == on got %q", v)
}
if !merge && ok {
t.Fatalf("expected archive_mode empty")
}
tk.Stop()
}
func TestPITR(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "stolon")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
baseBackupDir, err := ioutil.TempDir(dir, "basebackup")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
archiveBackupDir, err := ioutil.TempDir(dir, "archivebackup")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tstore := setupStore(t, dir)
defer tstore.Stop()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
clusterName := uuid.NewV4().String()
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
PGParameters: cluster.PGParameters{
"archive_mode": "on",
"archive_command": fmt.Sprintf("cp %%p %s/%%f", archiveBackupDir),
},
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tk, err := NewTestKeeper(t, dir, clusterName, pgSUUsername, pgSUPassword, pgReplUsername, pgReplPassword, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer tk.Stop()
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Wait for clusterView containing a master
_, err = WaitClusterDataWithMaster(sm, 30*time.Second)
if err != nil {
t.Fatal("expected a master in cluster view")
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitRole(common.RoleMaster, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := populate(t, tk); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := write(t, tk, 2, 2); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// ioutil.Tempfile already creates files with 0600 permissions
pgpass, err := ioutil.TempFile("", "pgpass")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
pgpass.WriteString(fmt.Sprintf("%s:%s:*:%s:%s\n", tk.pgListenAddress, tk.pgPort, tk.pgReplUsername, tk.pgReplPassword))
// Don't save the wal during the basebackup (-x). This to test that archive_command and restore command correctly work.
cmd := exec.Command("pg_basebackup", "-F", "tar", "-D", baseBackupDir, "-h", tk.pgListenAddress, "-p", tk.pgPort, "-U", tk.pgReplUsername)
cmd.Env = append(cmd.Env, fmt.Sprintf("PGPASSFILE=%s", pgpass.Name()))
t.Logf("execing cmd: %s", cmd)
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("error: %v, output: %s", err, string(out))
}
// Switch wal so they will be archived
if _, err := tk.db.Exec("select pg_switch_xlog()"); err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts.Stop()
// Delete the current cluster data
if err := tstore.store.Delete(filepath.Join(storePath, "clusterdata")); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Delete sentinel leader key to just speedup new election
if err := tstore.store.Delete(filepath.Join(storePath, common.SentinelLeaderKey)); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Now initialize a new cluster with the existing keeper
initialClusterSpec = &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModePITR),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
PITRConfig: &cluster.PITRConfig{
DataRestoreCommand: fmt.Sprintf("tar xvf %s/base.tar -C %%d", baseBackupDir),
ArchiveRecoverySettings: &cluster.ArchiveRecoverySettings{
RestoreCommand: fmt.Sprintf("cp %s/%%f %%p", archiveBackupDir),
},
},
}
initialClusterSpecFile, err = writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err = NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer ts.Stop()
if err := WaitClusterPhase(sm, cluster.ClusterPhaseNormal, 60*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
_, err = WaitClusterDataWithMaster(sm, 30*time.Second)
if err != nil {
t.Fatal("expected a master in cluster view")
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitRole(common.RoleMaster, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
c, err := getLines(t, tk)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if c != 1 {
t.Fatalf("wrong number of lines, want: %d, got: %d", 2, c)
}
}