func (bkr *Broker) deprovision(instanceID structs.ClusterID, details brokerapi.DeprovisionDetails, acceptsIncomplete bool) (async bool, err error) { logger := bkr.newLoggingSession("deprovision", lager.Data{"instanceID": instanceID}) defer logger.Info("done") if err = bkr.assertDeprovisionPrecondition(instanceID, details); err != nil { logger.Error("preconditions.error", err) return false, err } clusterState, err := bkr.state.LoadCluster(instanceID) if err != nil { logger.Error("load-cluster.error", err) return false, err } clusterModel := state.NewClusterModel(bkr.state, clusterState) err = bkr.scheduler.StopCluster(clusterModel) if err != nil { logger.Error("stop-cluster", err) return false, err } bkr.state.DeleteCluster(clusterModel.InstanceID()) bkr.router.RemoveClusterAssignment(clusterModel.InstanceID()) return false, nil }
func TestScheduler_allCells(t *testing.T) { t.Parallel() testPrefix := "TestScheduler_allCells" logger := testutil.NewTestLogger(testPrefix, t) schedulerConfig := config.Scheduler{ Cells: []*config.Cell{ &config.Cell{GUID: "cell-guid1"}, &config.Cell{GUID: "cell-guid2"}, }, Etcd: testutil.LocalEtcdConfig, } scheduler, err := NewScheduler(schedulerConfig, new(fakes.FakePatroni), logger) if err != nil { t.Fatalf("NewScheduler error: %v", err) } clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{InstanceID: "test"}) features := structs.ClusterFeatures{} plan, err := scheduler.newPlan(clusterModel, features) if err != nil { t.Fatalf("scheduler.newPlan error: %v", err) } if len(plan.availableCells) != 2 { t.Fatalf("Plan should have both cell cells") } if len(plan.allCells) != 2 { t.Fatalf("Plan should only have two cells") } }
func TestScheduler_filterCellsByCellGUIDs(t *testing.T) { t.Parallel() testPrefix := "TestScheduler_filterCellsByCellGUIDs" logger := testutil.NewTestLogger(testPrefix, t) schedulerConfig := config.Scheduler{ Cells: []*config.Cell{ &config.Cell{GUID: "cell-guid1"}, &config.Cell{GUID: "cell-guid2"}, }, Etcd: testutil.LocalEtcdConfig, } scheduler, err := NewScheduler(schedulerConfig, logger) if err != nil { t.Fatalf("NewScheduler error: %v", err) } features := structs.ClusterFeatures{ CellGUIDs: []string{"cell-guid1", "unknown-cell-guid"}, } clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{InstanceID: "test"}) plan, err := scheduler.newPlan(clusterModel, features) if err != nil { t.Fatalf("scheduler.newPlan error: %v", err) } if len(plan.availableCells) != 1 { t.Fatalf("Plan should only have one filtered cell") } if len(plan.allCells) != 2 { t.Fatalf("Plan should only have two cells") } }
func TestPlan_Steps_NewCluster_Default(t *testing.T) { t.Parallel() testPrefix := "TestPlan_Steps_NewCluster_Default" logger := testutil.NewTestLogger(testPrefix, t) schedulerConfig := config.Scheduler{ Cells: []*config.Cell{ &config.Cell{GUID: "cell1"}, &config.Cell{GUID: "cell2"}, &config.Cell{GUID: "cell3"}, &config.Cell{GUID: "cell4"}, }, Etcd: testutil.LocalEtcdConfig, } scheduler, err := NewScheduler(schedulerConfig, new(fakes.FakePatroni), logger) if err != nil { t.Fatalf("NewScheduler error: %v", err) } clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{}) plan, err := scheduler.newPlan(clusterModel, structs.ClusterFeatures{NodeCount: 2}) if err != nil { t.Fatalf("scheduler.newPlan error: %v", err) } expectedStepTypes := []string{"AddNode", "AddNode", "WaitForAllMembers", "WaitForLeader"} stepTypes := plan.stepTypes() if !reflect.DeepEqual(stepTypes, expectedStepTypes) { t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes) } }
func (bkr *Broker) update(instanceID structs.ClusterID, updateDetails brokerapi.UpdateDetails, acceptsIncomplete bool) (async bool, err error) { logger := bkr.newLoggingSession("update", lager.Data{"instanceID": instanceID}) defer logger.Info("done") features, err := structs.ClusterFeaturesFromParameters(updateDetails.Parameters) if err != nil { logger.Error("cluster-features", err) return false, err } if err := bkr.assertUpdatePrecondition(instanceID, features); err != nil { logger.Error("preconditions.error", err) return false, err } clusterState, err := bkr.state.LoadCluster(instanceID) if err != nil { logger.Error("load-cluster.error", err) return false, err } clusterModel := state.NewClusterModel(bkr.state, clusterState) go func() { err = bkr.scheduler.RunCluster(clusterModel, features) if err != nil { logger.Error("run-cluster", err) } }() return true, err }
func (bkr *Broker) lastOperation(instanceID structs.ClusterID) (resp brokerapi.LastOperationResponse, err error) { logger := bkr.newLoggingSession("last-opration", lager.Data{"instanceID": instanceID}) defer logger.Info("done") clusterState, err := bkr.state.LoadCluster(instanceID) if err != nil { logger.Error("load-cluster.error", err) return brokerapi.LastOperationResponse{State: brokerapi.LastOperationFailed, Description: err.Error()}, err } clusterModel := state.NewClusterModel(bkr.state, clusterState) return bkr.lastOperationFromSchedulingInfo(clusterModel.SchedulingInfo()) }
func (bkr *Broker) provision(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) { if details.ServiceID == "" && details.PlanID == "" { return bkr.Recreate(instanceID, details, acceptsIncomplete) } logger := bkr.newLoggingSession("provision", lager.Data{"instanceID": instanceID}) defer logger.Info("done") features, err := structs.ClusterFeaturesFromParameters(details.Parameters) if err != nil { logger.Error("cluster-features", err) return resp, false, err } if err = bkr.assertProvisionPrecondition(instanceID, features); err != nil { logger.Error("preconditions.error", err) return resp, false, err } port, err := bkr.router.AllocatePort() clusterState := bkr.initCluster(instanceID, port, details) clusterModel := state.NewClusterModel(bkr.state, clusterState) if bkr.callbacks.Configured() { bkr.callbacks.WriteRecreationData(clusterState.RecreationData()) data, err := bkr.callbacks.RestoreRecreationData(instanceID) if !reflect.DeepEqual(clusterState.RecreationData(), data) { logger.Error("recreation-data.failure", err) return resp, false, err } } // Continue processing in background // TODO: if error, store it into etcd; and last_operation_endpoint should look for errors first go func() { err := bkr.scheduler.RunCluster(clusterModel, features) if err != nil { logger.Error("run-cluster", err) return } err = bkr.router.AssignPortToCluster(clusterModel.InstanceID(), port) if err != nil { logger.Error("assign-port", err) } }() return resp, true, err }
func TestPlan_Steps_NewCluster_MoveEverything(t *testing.T) { t.Parallel() testPrefix := "TestPlan_Steps_NewCluster_MoveEverything" logger := testutil.NewTestLogger(testPrefix, t) schedulerConfig := config.Scheduler{ Cells: []*config.Cell{ &config.Cell{GUID: "cell1"}, &config.Cell{GUID: "cell2"}, }, Etcd: testutil.LocalEtcdConfig, } patroni := new(fakes.FakePatroni) patroni.ClusterLeaderStub = func(structs.ClusterID) (string, error) { return "a", nil } scheduler, err := NewScheduler(schedulerConfig, patroni, logger) if err != nil { t.Fatalf("NewScheduler error: %v", err) } clusterState := structs.ClusterState{ InstanceID: "test", Nodes: []*structs.Node{ &structs.Node{ID: "a", CellGUID: "cell-x-unavailable"}, &structs.Node{ID: "b", CellGUID: "cell-y-unavailable"}, }, } clusterFeatures := structs.ClusterFeatures{ NodeCount: 2, CellGUIDs: []string{"cell1", "cell2"}, } clusterModel := state.NewClusterModel(&state.StateEtcd{}, clusterState) plan, err := scheduler.newPlan(clusterModel, clusterFeatures) if err != nil { t.Fatalf("scheduler.newPlan error: %v", err) } expectedStepTypes := []string{"AddNode", "AddNode", "WaitForAllMembers", "RemoveNode(b)", "WaitForAllMembers", "FailoverFrom(a)", "RemoveNode(a)", "WaitForLeader"} stepTypes := plan.stepTypes() if !reflect.DeepEqual(stepTypes, expectedStepTypes) { t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes) } }
func TestPlan_Steps_NewCluster_MoveLeader(t *testing.T) { t.Parallel() testPrefix := "TestPlan_Steps_NewCluster_MoveLeader" logger := testutil.NewTestLogger(testPrefix, t) schedulerConfig := config.Scheduler{ Cells: []*config.Cell{ &config.Cell{GUID: "cell1"}, &config.Cell{GUID: "cell2"}, }, Etcd: testutil.LocalEtcdConfig, } scheduler, err := NewScheduler(schedulerConfig, logger) if err != nil { t.Fatalf("NewScheduler error: %v", err) } clusterState := structs.ClusterState{ InstanceID: "test", Nodes: []*structs.Node{ &structs.Node{ID: "a", CellGUID: "cell-unavailable", Role: state.LeaderRole}, &structs.Node{ID: "b", CellGUID: "cell2", Role: state.ReplicaRole}, }, } clusterFeatures := structs.ClusterFeatures{ NodeCount: 2, CellGUIDs: []string{"cell1", "cell2"}, } clusterModel := state.NewClusterModel(&state.StateEtcd{}, clusterState) plan, err := scheduler.newPlan(clusterModel, clusterFeatures) if err != nil { t.Fatalf("scheduler.newPlan error: %v", err) } expectedStepTypes := []string{"AddNode", "WaitForAllMembers", "RemoveLeader(a)", "WaitForAllMembers", "WaitForLeader"} stepTypes := plan.stepTypes() if !reflect.DeepEqual(stepTypes, expectedStepTypes) { t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes) } }
// Recreate service instance; invoked via Provision endpoint func (bkr *Broker) Recreate(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) { logger := bkr.newLoggingSession("recreate", lager.Data{}) defer logger.Info("stop") features, err := structs.ClusterFeaturesFromParameters(details.Parameters) if err != nil { logger.Error("cluster-features", err) return resp, false, err } if err = bkr.assertRecreatePrecondition(instanceID, features); err != nil { logger.Error("preconditions.error", err) return resp, false, err } recreationData, err := bkr.callbacks.RestoreRecreationData(instanceID) if err != nil { err = fmt.Errorf("Cannot recreate service from backup; unable to restore original service instance data: %s", err) return } clusterState := bkr.initClusterStateFromRecreationData(recreationData) clusterModel := state.NewClusterModel(bkr.state, clusterState) go func() { err := bkr.scheduler.RunCluster(clusterModel, features) if err != nil { logger.Error("run-cluster", err) return } err = bkr.router.AssignPortToCluster(clusterModel.InstanceID(), clusterModel.AllocatedPort()) if err != nil { logger.Error("assign-port", err) } }() return resp, true, err }
func (bkr *Broker) provision(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) { if details.ServiceID == "" && details.PlanID == "" { return bkr.Recreate(instanceID, details, acceptsIncomplete) } logger := bkr.newLoggingSession("provision", lager.Data{"instance-id": instanceID}) defer logger.Info("done") features, err := structs.ClusterFeaturesFromParameters(details.Parameters) if err != nil { logger.Error("cluster-features", err) return resp, false, err } if err = bkr.assertProvisionPrecondition(instanceID, features); err != nil { logger.Error("preconditions.error", err) return resp, false, err } port, err := bkr.router.AllocatePort() clusterState := bkr.initCluster(instanceID, port, details) clusterModel := state.NewClusterModel(bkr.state, clusterState) clusterModel.SchedulingMessage("Initializing...") if bkr.callbacks.Configured() { logger.Info("recreation-data.writing") bkr.callbacks.WriteRecreationData(clusterState.RecreationData()) logger.Info("recreation-data.restoring") data, err := bkr.callbacks.RestoreRecreationData(instanceID) if err != nil { logger.Error("recreation-data.save-failure.error", err) return resp, false, err } if !reflect.DeepEqual(clusterState.RecreationData(), data) { err = fmt.Errorf("Cluster recreation data was not saved successfully") logger.Error("recreation-data.save-failure.deep-equal", err) return resp, false, err } logger.Info("recreation-data.success") } var existingClusterData *structs.ClusterRecreationData if features.CloneFromServiceName != "" { if bkr.backups.BaseURI == "" { return resp, false, fmt.Errorf("Broker missing configuration backups.base_uri to support 'clone-from' feature") } // Confirm that backup can be found before continuing asynchronously logger.Info("lookup-service-name.start") existingClusterData, err = bkr.lookupClusterDataBackupByServiceInstanceName(details.SpaceGUID, features.CloneFromServiceName, logger) if err != nil { logger.Error("lookup-service-name.error", err) return resp, false, err } logger.Info("lookup-service-name.success") } // Continue processing in background go func() { logger.Info("async-begin") defer logger.Info("async-complete") if existingClusterData != nil { clusterModel.SchedulingMessage(fmt.Sprintf("Cloning existing database %s", existingClusterData.ServiceInstanceName)) if err := bkr.prepopulateDatabaseFromExistingClusterData(existingClusterData, instanceID, clusterModel, logger); err != nil { logger.Error("pre-populate-cluster", err) clusterModel.SchedulingError(fmt.Errorf("Unsuccessful pre-populating database from backup. Please contact administrator: %s", err.Error())) return } } if err := bkr.scheduler.RunCluster(clusterModel, features); err != nil { logger.Error("run-cluster", err) return } if err := bkr.router.AssignPortToCluster(instanceID, port); err != nil { logger.Error("assign-port", err) clusterModel.SchedulingError(fmt.Errorf("Unsuccessful mapping database to routing mesh. Please contact administrator: %s", err.Error())) return } bkr.fetchAndBackupServiceInstanceName(instanceID, &clusterState, logger) }() return resp, true, err }