func setupServers(t *testing.T, clusterName, dir string, numKeepers, numSentinels uint8, syncRepl bool, usePgrewind bool) (testKeepers, testSentinels, *TestStore) {
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
SynchronousReplication: cluster.BoolP(syncRepl),
UsePgrewind: cluster.BoolP(usePgrewind),
PGParameters: make(cluster.PGParameters),
}
return setupServersCustom(t, clusterName, dir, numKeepers, numSentinels, initialClusterSpec)
}
func TestInitialClusterSpec(t *testing.T) {
t.Parallel()
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
defer tstore.Stop()
clusterName := uuid.NewV4().String()
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
storePath := filepath.Join(common.StoreBasePath, clusterName)
sm := store.NewStoreManager(tstore.store, storePath)
initialClusterSpec := &cluster.ClusterSpec{
InitMode: cluster.ClusterInitModeP(cluster.ClusterInitModeNew),
SleepInterval: &cluster.Duration{Duration: 2 * time.Second},
FailInterval: &cluster.Duration{Duration: 5 * time.Second},
ConvergenceTimeout: &cluster.Duration{Duration: 30 * time.Second},
SynchronousReplication: cluster.BoolP(true),
}
initialClusterSpecFile, err := writeClusterSpec(dir, initialClusterSpec)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
ts, err := NewTestSentinel(t, dir, clusterName, tstore.storeBackend, storeEndpoints, fmt.Sprintf("--initial-cluster-spec=%s", initialClusterSpecFile))
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer ts.Stop()
if err := WaitClusterPhase(sm, cluster.ClusterPhaseInitializing, 60*time.Second); err != nil {
t.Fatal("expected cluster in initializing phase")
}
cd, _, err := sm.GetClusterData()
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if !*cd.Cluster.Spec.SynchronousReplication {
t.Fatal("expected cluster spec with SynchronousReplication enabled")
}
}
func TestUpdateClusterView(t *testing.T) {
tests := []struct {
cv *cluster.ClusterView
keepersState cluster.KeepersState
outCV *cluster.ClusterView
err error
}{
{
cv: cluster.NewClusterView(),
keepersState: nil,
outCV: cluster.NewClusterView(),
err: fmt.Errorf("cluster view at version 0 without a defined master. This shouldn't happen!"),
},
{
cv: &cluster.ClusterView{
Version: 1,
KeepersRole: cluster.KeepersRole{},
},
keepersState: nil,
outCV: &cluster.ClusterView{
Version: 1,
KeepersRole: cluster.KeepersRole{},
},
err: fmt.Errorf("cannot choose initial master, no keepers registered"),
},
// cluster initialization, one keeper
{
cv: &cluster.ClusterView{
Version: 1,
KeepersRole: cluster.KeepersRole{},
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{PGState: &cluster.PostgresState{Initialized: true}},
},
outCV: &cluster.ClusterView{
Version: 2,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
},
},
},
// cluster initialization, too many keepers
{
cv: &cluster.ClusterView{
Version: 1,
KeepersRole: cluster.KeepersRole{},
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{},
"02": &cluster.KeeperState{},
},
outCV: &cluster.ClusterView{
Version: 1,
KeepersRole: cluster.KeepersRole{},
},
err: fmt.Errorf("cannot choose initial master, more than 1 keeper registered"),
},
// cluster initialization, more then one keeper but InitWithMultipleKeepers == true
{
cv: &cluster.ClusterView{
Version: 1,
KeepersRole: cluster.KeepersRole{},
Config: &cluster.NilConfig{InitWithMultipleKeepers: cluster.BoolP(true)},
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{PGState: &cluster.PostgresState{Initialized: true}},
"02": &cluster.KeeperState{PGState: &cluster.PostgresState{Initialized: true}},
},
outCV: &cluster.ClusterView{
Version: 2,
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: ""},
},
Config: &cluster.NilConfig{InitWithMultipleKeepers: cluster.BoolP(true)},
},
},
// One master and one standby, both healthy: no change from previous cv
{
cv: &cluster.ClusterView{
Version: 1,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: "01"},
},
ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"},
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "01",
PGPort: "01",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
"02": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "02",
PGPort: "02",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
},
outCV: &cluster.ClusterView{
Version: 1,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: "01"},
},
ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"},
},
},
// One master and one standby, master not healthy: standby elected as new master
{
cv: &cluster.ClusterView{
Version: 1,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: "01"},
},
ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"},
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "01",
PGPort: "01",
ErrorStartTime: time.Unix(0, 0),
Healthy: false,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
"02": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "02",
PGPort: "02",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
},
outCV: &cluster.ClusterView{
Version: 2,
Master: "02",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: ""},
},
ProxyConf: nil,
},
},
// From the previous test, new master (02) converged. Old master setup to follow new master (02).
{
cv: &cluster.ClusterView{
Version: 2,
Master: "02",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: ""},
},
ProxyConf: nil,
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "01",
PGPort: "01",
ErrorStartTime: time.Unix(0, 0),
Healthy: false,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
"02": &cluster.KeeperState{
ClusterViewVersion: 2,
PGListenAddress: "02",
PGPort: "02",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
},
outCV: &cluster.ClusterView{
Version: 3,
Master: "02",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: "02"},
"02": &cluster.KeeperRole{ID: "02", Follow: ""},
},
ProxyConf: &cluster.ProxyConf{Host: "02", Port: "02"},
},
},
// One master and one standby, master not healthy, standby with old
// clusterview: no standby elected as new master.
{
cv: &cluster.ClusterView{
Version: 2,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: "01"},
},
ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"},
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{
ClusterViewVersion: 2,
PGListenAddress: "01",
PGPort: "01",
ErrorStartTime: time.Unix(0, 0),
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
"02": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "02",
PGPort: "02",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
},
outCV: &cluster.ClusterView{
Version: 2,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: "01"},
},
ProxyConf: &cluster.ProxyConf{Host: "01", Port: "01"},
},
},
// One master and one standby, master not converged to current
// cv: standby elected as new master.
{
cv: &cluster.ClusterView{
Version: 2,
Master: "01",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: "01"},
},
ProxyConf: nil,
},
keepersState: cluster.KeepersState{
"01": &cluster.KeeperState{
ClusterViewVersion: 1,
PGListenAddress: "01",
PGPort: "01",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
"02": &cluster.KeeperState{
ClusterViewVersion: 2,
PGListenAddress: "02",
PGPort: "02",
ErrorStartTime: time.Time{},
Healthy: true,
PGState: &cluster.PostgresState{
TimelineID: 0,
},
},
},
outCV: &cluster.ClusterView{
Version: 3,
Master: "02",
KeepersRole: cluster.KeepersRole{
"01": &cluster.KeeperRole{ID: "01", Follow: ""},
"02": &cluster.KeeperRole{ID: "02", Follow: ""},
},
ProxyConf: nil,
},
},
}
for i, tt := range tests {
var s *Sentinel
if tt.cv.Config == nil {
s = &Sentinel{id: "id", clusterConfig: cluster.NewDefaultConfig()}
} else {
s = &Sentinel{id: "id", clusterConfig: tt.cv.Config.ToConfig()}
}
outCV, err := s.updateClusterView(tt.cv, tt.keepersState)
t.Logf("test #%d", i)
t.Logf(spew.Sprintf("outCV: %#v", outCV))
if tt.err != nil {
if err == nil {
t.Errorf("got no error, wanted error: %v", tt.err)
} else if tt.err.Error() != err.Error() {
t.Errorf("got error: %v, wanted error: %v", err, tt.err)
}
} else {
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !outCV.Equals(tt.outCV) {
t.Errorf(spew.Sprintf("#%d: wrong outCV: got: %#v, want: %#v", i, outCV, tt.outCV))
}
}
}
}
func setupServers(t *testing.T, dir string, numKeepers, numSentinels uint8, syncRepl bool) ([]*TestKeeper, []*TestSentinel, *TestEtcd) {
te, err := NewTestEtcd(dir)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := te.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := te.WaitUp(10 * time.Second); err != nil {
t.Fatalf("error waiting on etcd up: %v", err)
}
etcdEndpoints := fmt.Sprintf("http://%s:%s", te.listenAddress, te.port)
clusterName := uuid.NewV4().String()
etcdPath := filepath.Join(common.EtcdBasePath, clusterName)
e, err := etcdm.NewEtcdManager(etcdEndpoints, etcdPath, common.DefaultEtcdRequestTimeout)
if err != nil {
t.Fatalf("cannot create etcd manager: %v", err)
}
// TODO(sgotti) change this to a call to the sentinel to change the
// cluster config (when the sentinel's code is done)
e.SetClusterData(cluster.KeepersState{},
&cluster.ClusterView{
Version: 1,
Config: &cluster.NilConfig{
SleepInterval: cluster.DurationP(5 * time.Second),
KeeperFailInterval: cluster.DurationP(10 * time.Second),
SynchronousReplication: cluster.BoolP(syncRepl),
},
}, 0)
tks := []*TestKeeper{}
tss := []*TestSentinel{}
tk, err := NewTestKeeper(dir, clusterName, etcdEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks = append(tks, tk)
fmt.Printf("tk: %v\n", tk)
// Start sentinels
for i := uint8(0); i < numSentinels; i++ {
ts, err := NewTestSentinel(dir, clusterName, etcdEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tss = append(tss, ts)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitRole(common.MasterRole, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Wait for clusterView containing tk as master
if err := WaitClusterViewMaster(tk.id, e, 30*time.Second); err != nil {
t.Fatalf("expected master %q in cluster view", tk.id)
}
// Start standbys
for i := uint8(1); i < numKeepers; i++ {
tk, err := NewTestKeeper(dir, clusterName, etcdEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitRole(common.StandbyRole, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks = append(tks, tk)
}
return tks, tss, te
}
func TestInitWithMultipleKeepers(t *testing.T) {
dir, err := ioutil.TempDir("", "stolon")
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
defer os.RemoveAll(dir)
tstore := setupStore(t, dir)
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
clusterName := uuid.NewV4().String()
storePath := filepath.Join(common.StoreBasePath, clusterName)
kvstore, err := store.NewStore(tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("cannot create store: %v", err)
}
e := store.NewStoreManager(kvstore, storePath)
// TODO(sgotti) change this to a call to the sentinel to change the
// cluster config (when the sentinel's code is done)
e.SetClusterData(cluster.KeepersState{},
&cluster.ClusterView{
Version: 1,
Config: &cluster.NilConfig{
SleepInterval: &cluster.Duration{5 * time.Second},
KeeperFailInterval: &cluster.Duration{10 * time.Second},
InitWithMultipleKeepers: cluster.BoolP(true),
},
}, nil)
tks := []*TestKeeper{}
tss := []*TestSentinel{}
// Start 3 keepers
for i := uint8(0); i < 3; i++ {
tk, err := NewTestKeeper(dir, clusterName, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks = append(tks, tk)
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
}
// Start 2 sentinels
for i := uint8(0); i < 2; i++ {
ts, err := NewTestSentinel(dir, clusterName, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tss = append(tss, ts)
}
defer shutdown(tks, tss, tstore)
// Wait for clusterView containing a master
if err := WaitClusterViewWithMaster(e, 30*time.Second); err != nil {
t.Fatal("expected a master in cluster view")
}
}
func setupServers(t *testing.T, dir string, numKeepers, numSentinels uint8, syncRepl bool) ([]*TestKeeper, []*TestSentinel, *TestStore) {
tstore := setupStore(t, dir)
storeEndpoints := fmt.Sprintf("%s:%s", tstore.listenAddress, tstore.port)
clusterName := uuid.NewV4().String()
storePath := filepath.Join(common.StoreBasePath, clusterName)
kvstore, err := store.NewStore(tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("cannot create store: %v", err)
}
e := store.NewStoreManager(kvstore, storePath)
// TODO(sgotti) change this to a call to the sentinel to change the
// cluster config (when the sentinel's code is done)
e.SetClusterData(cluster.KeepersState{},
&cluster.ClusterView{
Version: 1,
Config: &cluster.NilConfig{
SleepInterval: &cluster.Duration{5 * time.Second},
KeeperFailInterval: &cluster.Duration{10 * time.Second},
SynchronousReplication: cluster.BoolP(syncRepl),
},
}, nil)
tks := []*TestKeeper{}
tss := []*TestSentinel{}
tk, err := NewTestKeeper(dir, clusterName, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks = append(tks, tk)
fmt.Printf("tk: %v\n", tk)
// Start sentinels
for i := uint8(0); i < numSentinels; i++ {
ts, err := NewTestSentinel(dir, clusterName, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := ts.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tss = append(tss, ts)
}
// Start first keeper
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitRole(common.MasterRole, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Wait for clusterView containing tk as master
if err := WaitClusterViewMaster(tk.id, e, 30*time.Second); err != nil {
t.Fatalf("expected master %q in cluster view", tk.id)
}
// Start other keepers
for i := uint8(1); i < numKeepers; i++ {
tk, err := NewTestKeeper(dir, clusterName, tstore.storeBackend, storeEndpoints)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.Start(); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if err := tk.WaitDBUp(60 * time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
// Wait for clusterView containing tk as standby
if err := tk.WaitRole(common.StandbyRole, 30*time.Second); err != nil {
t.Fatalf("unexpected err: %v", err)
}
tks = append(tks, tk)
}
return tks, tss, tstore
}