func (bkr *Broker) deprovision(instanceID structs.ClusterID, details brokerapi.DeprovisionDetails, acceptsIncomplete bool) (async bool, err error) {
logger := bkr.newLoggingSession("deprovision", lager.Data{"instanceID": instanceID})
defer logger.Info("done")
if err = bkr.assertDeprovisionPrecondition(instanceID, details); err != nil {
logger.Error("preconditions.error", err)
return false, err
}
clusterState, err := bkr.state.LoadCluster(instanceID)
if err != nil {
logger.Error("load-cluster.error", err)
return false, err
}
clusterModel := state.NewClusterModel(bkr.state, clusterState)
err = bkr.scheduler.StopCluster(clusterModel)
if err != nil {
logger.Error("stop-cluster", err)
return false, err
}
bkr.state.DeleteCluster(clusterModel.InstanceID())
bkr.router.RemoveClusterAssignment(clusterModel.InstanceID())
return false, nil
}
func TestScheduler_allCells(t *testing.T) {
t.Parallel()
testPrefix := "TestScheduler_allCells"
logger := testutil.NewTestLogger(testPrefix, t)
schedulerConfig := config.Scheduler{
Cells: []*config.Cell{
&config.Cell{GUID: "cell-guid1"},
&config.Cell{GUID: "cell-guid2"},
},
Etcd: testutil.LocalEtcdConfig,
}
scheduler, err := NewScheduler(schedulerConfig, new(fakes.FakePatroni), logger)
if err != nil {
t.Fatalf("NewScheduler error: %v", err)
}
clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{InstanceID: "test"})
features := structs.ClusterFeatures{}
plan, err := scheduler.newPlan(clusterModel, features)
if err != nil {
t.Fatalf("scheduler.newPlan error: %v", err)
}
if len(plan.availableCells) != 2 {
t.Fatalf("Plan should have both cell cells")
}
if len(plan.allCells) != 2 {
t.Fatalf("Plan should only have two cells")
}
}
func TestScheduler_filterCellsByCellGUIDs(t *testing.T) {
t.Parallel()
testPrefix := "TestScheduler_filterCellsByCellGUIDs"
logger := testutil.NewTestLogger(testPrefix, t)
schedulerConfig := config.Scheduler{
Cells: []*config.Cell{
&config.Cell{GUID: "cell-guid1"},
&config.Cell{GUID: "cell-guid2"},
},
Etcd: testutil.LocalEtcdConfig,
}
scheduler, err := NewScheduler(schedulerConfig, logger)
if err != nil {
t.Fatalf("NewScheduler error: %v", err)
}
features := structs.ClusterFeatures{
CellGUIDs: []string{"cell-guid1", "unknown-cell-guid"},
}
clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{InstanceID: "test"})
plan, err := scheduler.newPlan(clusterModel, features)
if err != nil {
t.Fatalf("scheduler.newPlan error: %v", err)
}
if len(plan.availableCells) != 1 {
t.Fatalf("Plan should only have one filtered cell")
}
if len(plan.allCells) != 2 {
t.Fatalf("Plan should only have two cells")
}
}
func TestPlan_Steps_NewCluster_Default(t *testing.T) {
t.Parallel()
testPrefix := "TestPlan_Steps_NewCluster_Default"
logger := testutil.NewTestLogger(testPrefix, t)
schedulerConfig := config.Scheduler{
Cells: []*config.Cell{
&config.Cell{GUID: "cell1"},
&config.Cell{GUID: "cell2"},
&config.Cell{GUID: "cell3"},
&config.Cell{GUID: "cell4"},
},
Etcd: testutil.LocalEtcdConfig,
}
scheduler, err := NewScheduler(schedulerConfig, new(fakes.FakePatroni), logger)
if err != nil {
t.Fatalf("NewScheduler error: %v", err)
}
clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{})
plan, err := scheduler.newPlan(clusterModel, structs.ClusterFeatures{NodeCount: 2})
if err != nil {
t.Fatalf("scheduler.newPlan error: %v", err)
}
expectedStepTypes := []string{"AddNode", "AddNode", "WaitForAllMembers", "WaitForLeader"}
stepTypes := plan.stepTypes()
if !reflect.DeepEqual(stepTypes, expectedStepTypes) {
t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes)
}
}
func (bkr *Broker) update(instanceID structs.ClusterID, updateDetails brokerapi.UpdateDetails, acceptsIncomplete bool) (async bool, err error) {
logger := bkr.newLoggingSession("update", lager.Data{"instanceID": instanceID})
defer logger.Info("done")
features, err := structs.ClusterFeaturesFromParameters(updateDetails.Parameters)
if err != nil {
logger.Error("cluster-features", err)
return false, err
}
if err := bkr.assertUpdatePrecondition(instanceID, features); err != nil {
logger.Error("preconditions.error", err)
return false, err
}
clusterState, err := bkr.state.LoadCluster(instanceID)
if err != nil {
logger.Error("load-cluster.error", err)
return false, err
}
clusterModel := state.NewClusterModel(bkr.state, clusterState)
go func() {
err = bkr.scheduler.RunCluster(clusterModel, features)
if err != nil {
logger.Error("run-cluster", err)
}
}()
return true, err
}
func (bkr *Broker) lastOperation(instanceID structs.ClusterID) (resp brokerapi.LastOperationResponse, err error) {
logger := bkr.newLoggingSession("last-opration", lager.Data{"instanceID": instanceID})
defer logger.Info("done")
clusterState, err := bkr.state.LoadCluster(instanceID)
if err != nil {
logger.Error("load-cluster.error", err)
return brokerapi.LastOperationResponse{State: brokerapi.LastOperationFailed, Description: err.Error()}, err
}
clusterModel := state.NewClusterModel(bkr.state, clusterState)
return bkr.lastOperationFromSchedulingInfo(clusterModel.SchedulingInfo())
}
func (bkr *Broker) provision(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) {
if details.ServiceID == "" && details.PlanID == "" {
return bkr.Recreate(instanceID, details, acceptsIncomplete)
}
logger := bkr.newLoggingSession("provision", lager.Data{"instanceID": instanceID})
defer logger.Info("done")
features, err := structs.ClusterFeaturesFromParameters(details.Parameters)
if err != nil {
logger.Error("cluster-features", err)
return resp, false, err
}
if err = bkr.assertProvisionPrecondition(instanceID, features); err != nil {
logger.Error("preconditions.error", err)
return resp, false, err
}
port, err := bkr.router.AllocatePort()
clusterState := bkr.initCluster(instanceID, port, details)
clusterModel := state.NewClusterModel(bkr.state, clusterState)
if bkr.callbacks.Configured() {
bkr.callbacks.WriteRecreationData(clusterState.RecreationData())
data, err := bkr.callbacks.RestoreRecreationData(instanceID)
if !reflect.DeepEqual(clusterState.RecreationData(), data) {
logger.Error("recreation-data.failure", err)
return resp, false, err
}
}
// Continue processing in background
// TODO: if error, store it into etcd; and last_operation_endpoint should look for errors first
go func() {
err := bkr.scheduler.RunCluster(clusterModel, features)
if err != nil {
logger.Error("run-cluster", err)
return
}
err = bkr.router.AssignPortToCluster(clusterModel.InstanceID(), port)
if err != nil {
logger.Error("assign-port", err)
}
}()
return resp, true, err
}
func TestPlan_Steps_NewCluster_MoveEverything(t *testing.T) {
t.Parallel()
testPrefix := "TestPlan_Steps_NewCluster_MoveEverything"
logger := testutil.NewTestLogger(testPrefix, t)
schedulerConfig := config.Scheduler{
Cells: []*config.Cell{
&config.Cell{GUID: "cell1"},
&config.Cell{GUID: "cell2"},
},
Etcd: testutil.LocalEtcdConfig,
}
patroni := new(fakes.FakePatroni)
patroni.ClusterLeaderStub = func(structs.ClusterID) (string, error) { return "a", nil }
scheduler, err := NewScheduler(schedulerConfig, patroni, logger)
if err != nil {
t.Fatalf("NewScheduler error: %v", err)
}
clusterState := structs.ClusterState{
InstanceID: "test",
Nodes: []*structs.Node{
&structs.Node{ID: "a", CellGUID: "cell-x-unavailable"},
&structs.Node{ID: "b", CellGUID: "cell-y-unavailable"},
},
}
clusterFeatures := structs.ClusterFeatures{
NodeCount: 2,
CellGUIDs: []string{"cell1", "cell2"},
}
clusterModel := state.NewClusterModel(&state.StateEtcd{}, clusterState)
plan, err := scheduler.newPlan(clusterModel, clusterFeatures)
if err != nil {
t.Fatalf("scheduler.newPlan error: %v", err)
}
expectedStepTypes := []string{"AddNode", "AddNode", "WaitForAllMembers", "RemoveNode(b)", "WaitForAllMembers", "FailoverFrom(a)", "RemoveNode(a)", "WaitForLeader"}
stepTypes := plan.stepTypes()
if !reflect.DeepEqual(stepTypes, expectedStepTypes) {
t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes)
}
}
func TestPlan_Steps_NewCluster_MoveLeader(t *testing.T) {
t.Parallel()
testPrefix := "TestPlan_Steps_NewCluster_MoveLeader"
logger := testutil.NewTestLogger(testPrefix, t)
schedulerConfig := config.Scheduler{
Cells: []*config.Cell{
&config.Cell{GUID: "cell1"},
&config.Cell{GUID: "cell2"},
},
Etcd: testutil.LocalEtcdConfig,
}
scheduler, err := NewScheduler(schedulerConfig, logger)
if err != nil {
t.Fatalf("NewScheduler error: %v", err)
}
clusterState := structs.ClusterState{
InstanceID: "test",
Nodes: []*structs.Node{
&structs.Node{ID: "a", CellGUID: "cell-unavailable", Role: state.LeaderRole},
&structs.Node{ID: "b", CellGUID: "cell2", Role: state.ReplicaRole},
},
}
clusterFeatures := structs.ClusterFeatures{
NodeCount: 2,
CellGUIDs: []string{"cell1", "cell2"},
}
clusterModel := state.NewClusterModel(&state.StateEtcd{}, clusterState)
plan, err := scheduler.newPlan(clusterModel, clusterFeatures)
if err != nil {
t.Fatalf("scheduler.newPlan error: %v", err)
}
expectedStepTypes := []string{"AddNode", "WaitForAllMembers", "RemoveLeader(a)", "WaitForAllMembers", "WaitForLeader"}
stepTypes := plan.stepTypes()
if !reflect.DeepEqual(stepTypes, expectedStepTypes) {
t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes)
}
}
// Recreate service instance; invoked via Provision endpoint
func (bkr *Broker) Recreate(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) {
logger := bkr.newLoggingSession("recreate", lager.Data{})
defer logger.Info("stop")
features, err := structs.ClusterFeaturesFromParameters(details.Parameters)
if err != nil {
logger.Error("cluster-features", err)
return resp, false, err
}
if err = bkr.assertRecreatePrecondition(instanceID, features); err != nil {
logger.Error("preconditions.error", err)
return resp, false, err
}
recreationData, err := bkr.callbacks.RestoreRecreationData(instanceID)
if err != nil {
err = fmt.Errorf("Cannot recreate service from backup; unable to restore original service instance data: %s", err)
return
}
clusterState := bkr.initClusterStateFromRecreationData(recreationData)
clusterModel := state.NewClusterModel(bkr.state, clusterState)
go func() {
err := bkr.scheduler.RunCluster(clusterModel, features)
if err != nil {
logger.Error("run-cluster", err)
return
}
err = bkr.router.AssignPortToCluster(clusterModel.InstanceID(), clusterModel.AllocatedPort())
if err != nil {
logger.Error("assign-port", err)
}
}()
return resp, true, err
}
func (bkr *Broker) provision(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) {
if details.ServiceID == "" && details.PlanID == "" {
return bkr.Recreate(instanceID, details, acceptsIncomplete)
}
logger := bkr.newLoggingSession("provision", lager.Data{"instance-id": instanceID})
defer logger.Info("done")
features, err := structs.ClusterFeaturesFromParameters(details.Parameters)
if err != nil {
logger.Error("cluster-features", err)
return resp, false, err
}
if err = bkr.assertProvisionPrecondition(instanceID, features); err != nil {
logger.Error("preconditions.error", err)
return resp, false, err
}
port, err := bkr.router.AllocatePort()
clusterState := bkr.initCluster(instanceID, port, details)
clusterModel := state.NewClusterModel(bkr.state, clusterState)
clusterModel.SchedulingMessage("Initializing...")
if bkr.callbacks.Configured() {
logger.Info("recreation-data.writing")
bkr.callbacks.WriteRecreationData(clusterState.RecreationData())
logger.Info("recreation-data.restoring")
data, err := bkr.callbacks.RestoreRecreationData(instanceID)
if err != nil {
logger.Error("recreation-data.save-failure.error", err)
return resp, false, err
}
if !reflect.DeepEqual(clusterState.RecreationData(), data) {
err = fmt.Errorf("Cluster recreation data was not saved successfully")
logger.Error("recreation-data.save-failure.deep-equal", err)
return resp, false, err
}
logger.Info("recreation-data.success")
}
var existingClusterData *structs.ClusterRecreationData
if features.CloneFromServiceName != "" {
if bkr.backups.BaseURI == "" {
return resp, false, fmt.Errorf("Broker missing configuration backups.base_uri to support 'clone-from' feature")
}
// Confirm that backup can be found before continuing asynchronously
logger.Info("lookup-service-name.start")
existingClusterData, err = bkr.lookupClusterDataBackupByServiceInstanceName(details.SpaceGUID, features.CloneFromServiceName, logger)
if err != nil {
logger.Error("lookup-service-name.error", err)
return resp, false, err
}
logger.Info("lookup-service-name.success")
}
// Continue processing in background
go func() {
logger.Info("async-begin")
defer logger.Info("async-complete")
if existingClusterData != nil {
clusterModel.SchedulingMessage(fmt.Sprintf("Cloning existing database %s", existingClusterData.ServiceInstanceName))
if err := bkr.prepopulateDatabaseFromExistingClusterData(existingClusterData, instanceID, clusterModel, logger); err != nil {
logger.Error("pre-populate-cluster", err)
clusterModel.SchedulingError(fmt.Errorf("Unsuccessful pre-populating database from backup. Please contact administrator: %s", err.Error()))
return
}
}
if err := bkr.scheduler.RunCluster(clusterModel, features); err != nil {
logger.Error("run-cluster", err)
return
}
if err := bkr.router.AssignPortToCluster(instanceID, port); err != nil {
logger.Error("assign-port", err)
clusterModel.SchedulingError(fmt.Errorf("Unsuccessful mapping database to routing mesh. Please contact administrator: %s", err.Error()))
return
}
bkr.fetchAndBackupServiceInstanceName(instanceID, &clusterState, logger)
}()
return resp, true, err
}