func (bkr *Broker) deprovision(instanceID structs.ClusterID, details brokerapi.DeprovisionDetails, acceptsIncomplete bool) (async bool, err error) {
	logger := bkr.newLoggingSession("deprovision", lager.Data{"instanceID": instanceID})
	defer logger.Info("done")

	if err = bkr.assertDeprovisionPrecondition(instanceID, details); err != nil {
		logger.Error("preconditions.error", err)
		return false, err
	}

	clusterState, err := bkr.state.LoadCluster(instanceID)
	if err != nil {
		logger.Error("load-cluster.error", err)
		return false, err
	}

	clusterModel := state.NewClusterModel(bkr.state, clusterState)
	err = bkr.scheduler.StopCluster(clusterModel)
	if err != nil {
		logger.Error("stop-cluster", err)
		return false, err
	}

	bkr.state.DeleteCluster(clusterModel.InstanceID())
	bkr.router.RemoveClusterAssignment(clusterModel.InstanceID())

	return false, nil
}
func TestScheduler_allCells(t *testing.T) {
	t.Parallel()

	testPrefix := "TestScheduler_allCells"
	logger := testutil.NewTestLogger(testPrefix, t)

	schedulerConfig := config.Scheduler{
		Cells: []*config.Cell{
			&config.Cell{GUID: "cell-guid1"},
			&config.Cell{GUID: "cell-guid2"},
		},
		Etcd: testutil.LocalEtcdConfig,
	}
	scheduler, err := NewScheduler(schedulerConfig, new(fakes.FakePatroni), logger)
	if err != nil {
		t.Fatalf("NewScheduler error: %v", err)
	}

	clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{InstanceID: "test"})
	features := structs.ClusterFeatures{}
	plan, err := scheduler.newPlan(clusterModel, features)
	if err != nil {
		t.Fatalf("scheduler.newPlan error: %v", err)
	}

	if len(plan.availableCells) != 2 {
		t.Fatalf("Plan should have both cell cells")
	}
	if len(plan.allCells) != 2 {
		t.Fatalf("Plan should only have two cells")
	}
}
func TestScheduler_filterCellsByCellGUIDs(t *testing.T) {
	t.Parallel()

	testPrefix := "TestScheduler_filterCellsByCellGUIDs"
	logger := testutil.NewTestLogger(testPrefix, t)

	schedulerConfig := config.Scheduler{
		Cells: []*config.Cell{
			&config.Cell{GUID: "cell-guid1"},
			&config.Cell{GUID: "cell-guid2"},
		},
		Etcd: testutil.LocalEtcdConfig,
	}
	scheduler, err := NewScheduler(schedulerConfig, logger)
	if err != nil {
		t.Fatalf("NewScheduler error: %v", err)
	}

	features := structs.ClusterFeatures{
		CellGUIDs: []string{"cell-guid1", "unknown-cell-guid"},
	}
	clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{InstanceID: "test"})
	plan, err := scheduler.newPlan(clusterModel, features)
	if err != nil {
		t.Fatalf("scheduler.newPlan error: %v", err)
	}

	if len(plan.availableCells) != 1 {
		t.Fatalf("Plan should only have one filtered cell")
	}
	if len(plan.allCells) != 2 {
		t.Fatalf("Plan should only have two cells")
	}
}
func TestPlan_Steps_NewCluster_Default(t *testing.T) {
	t.Parallel()

	testPrefix := "TestPlan_Steps_NewCluster_Default"
	logger := testutil.NewTestLogger(testPrefix, t)

	schedulerConfig := config.Scheduler{
		Cells: []*config.Cell{
			&config.Cell{GUID: "cell1"},
			&config.Cell{GUID: "cell2"},
			&config.Cell{GUID: "cell3"},
			&config.Cell{GUID: "cell4"},
		},
		Etcd: testutil.LocalEtcdConfig,
	}
	scheduler, err := NewScheduler(schedulerConfig, new(fakes.FakePatroni), logger)
	if err != nil {
		t.Fatalf("NewScheduler error: %v", err)
	}

	clusterModel := state.NewClusterModel(&state.StateEtcd{}, structs.ClusterState{})
	plan, err := scheduler.newPlan(clusterModel, structs.ClusterFeatures{NodeCount: 2})
	if err != nil {
		t.Fatalf("scheduler.newPlan error: %v", err)
	}
	expectedStepTypes := []string{"AddNode", "AddNode", "WaitForAllMembers", "WaitForLeader"}
	stepTypes := plan.stepTypes()
	if !reflect.DeepEqual(stepTypes, expectedStepTypes) {
		t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes)
	}
}
func (bkr *Broker) update(instanceID structs.ClusterID, updateDetails brokerapi.UpdateDetails, acceptsIncomplete bool) (async bool, err error) {
	logger := bkr.newLoggingSession("update", lager.Data{"instanceID": instanceID})
	defer logger.Info("done")

	features, err := structs.ClusterFeaturesFromParameters(updateDetails.Parameters)
	if err != nil {
		logger.Error("cluster-features", err)
		return false, err
	}

	if err := bkr.assertUpdatePrecondition(instanceID, features); err != nil {
		logger.Error("preconditions.error", err)
		return false, err
	}

	clusterState, err := bkr.state.LoadCluster(instanceID)
	if err != nil {
		logger.Error("load-cluster.error", err)
		return false, err
	}
	clusterModel := state.NewClusterModel(bkr.state, clusterState)

	go func() {
		err = bkr.scheduler.RunCluster(clusterModel, features)
		if err != nil {
			logger.Error("run-cluster", err)
		}
	}()
	return true, err
}
func (bkr *Broker) lastOperation(instanceID structs.ClusterID) (resp brokerapi.LastOperationResponse, err error) {
	logger := bkr.newLoggingSession("last-opration", lager.Data{"instanceID": instanceID})
	defer logger.Info("done")

	clusterState, err := bkr.state.LoadCluster(instanceID)
	if err != nil {
		logger.Error("load-cluster.error", err)
		return brokerapi.LastOperationResponse{State: brokerapi.LastOperationFailed, Description: err.Error()}, err
	}
	clusterModel := state.NewClusterModel(bkr.state, clusterState)
	return bkr.lastOperationFromSchedulingInfo(clusterModel.SchedulingInfo())
}
func (bkr *Broker) provision(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) {
	if details.ServiceID == "" && details.PlanID == "" {
		return bkr.Recreate(instanceID, details, acceptsIncomplete)
	}

	logger := bkr.newLoggingSession("provision", lager.Data{"instanceID": instanceID})
	defer logger.Info("done")

	features, err := structs.ClusterFeaturesFromParameters(details.Parameters)
	if err != nil {
		logger.Error("cluster-features", err)
		return resp, false, err
	}

	if err = bkr.assertProvisionPrecondition(instanceID, features); err != nil {
		logger.Error("preconditions.error", err)
		return resp, false, err
	}

	port, err := bkr.router.AllocatePort()
	clusterState := bkr.initCluster(instanceID, port, details)
	clusterModel := state.NewClusterModel(bkr.state, clusterState)

	if bkr.callbacks.Configured() {
		bkr.callbacks.WriteRecreationData(clusterState.RecreationData())
		data, err := bkr.callbacks.RestoreRecreationData(instanceID)
		if !reflect.DeepEqual(clusterState.RecreationData(), data) {
			logger.Error("recreation-data.failure", err)
			return resp, false, err
		}
	}

	// Continue processing in background
	// TODO: if error, store it into etcd; and last_operation_endpoint should look for errors first
	go func() {
		err := bkr.scheduler.RunCluster(clusterModel, features)
		if err != nil {
			logger.Error("run-cluster", err)
			return
		}

		err = bkr.router.AssignPortToCluster(clusterModel.InstanceID(), port)
		if err != nil {
			logger.Error("assign-port", err)
		}
	}()
	return resp, true, err
}
func TestPlan_Steps_NewCluster_MoveEverything(t *testing.T) {
	t.Parallel()

	testPrefix := "TestPlan_Steps_NewCluster_MoveEverything"
	logger := testutil.NewTestLogger(testPrefix, t)

	schedulerConfig := config.Scheduler{
		Cells: []*config.Cell{
			&config.Cell{GUID: "cell1"},
			&config.Cell{GUID: "cell2"},
		},
		Etcd: testutil.LocalEtcdConfig,
	}
	patroni := new(fakes.FakePatroni)
	patroni.ClusterLeaderStub = func(structs.ClusterID) (string, error) { return "a", nil }
	scheduler, err := NewScheduler(schedulerConfig, patroni, logger)
	if err != nil {
		t.Fatalf("NewScheduler error: %v", err)
	}

	clusterState := structs.ClusterState{
		InstanceID: "test",
		Nodes: []*structs.Node{
			&structs.Node{ID: "a", CellGUID: "cell-x-unavailable"},
			&structs.Node{ID: "b", CellGUID: "cell-y-unavailable"},
		},
	}
	clusterFeatures := structs.ClusterFeatures{
		NodeCount: 2,
		CellGUIDs: []string{"cell1", "cell2"},
	}
	clusterModel := state.NewClusterModel(&state.StateEtcd{}, clusterState)
	plan, err := scheduler.newPlan(clusterModel, clusterFeatures)
	if err != nil {
		t.Fatalf("scheduler.newPlan error: %v", err)
	}
	expectedStepTypes := []string{"AddNode", "AddNode", "WaitForAllMembers", "RemoveNode(b)", "WaitForAllMembers", "FailoverFrom(a)", "RemoveNode(a)", "WaitForLeader"}
	stepTypes := plan.stepTypes()
	if !reflect.DeepEqual(stepTypes, expectedStepTypes) {
		t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes)
	}
}
func TestPlan_Steps_NewCluster_MoveLeader(t *testing.T) {
	t.Parallel()

	testPrefix := "TestPlan_Steps_NewCluster_MoveLeader"
	logger := testutil.NewTestLogger(testPrefix, t)

	schedulerConfig := config.Scheduler{
		Cells: []*config.Cell{
			&config.Cell{GUID: "cell1"},
			&config.Cell{GUID: "cell2"},
		},
		Etcd: testutil.LocalEtcdConfig,
	}
	scheduler, err := NewScheduler(schedulerConfig, logger)
	if err != nil {
		t.Fatalf("NewScheduler error: %v", err)
	}

	clusterState := structs.ClusterState{
		InstanceID: "test",
		Nodes: []*structs.Node{
			&structs.Node{ID: "a", CellGUID: "cell-unavailable", Role: state.LeaderRole},
			&structs.Node{ID: "b", CellGUID: "cell2", Role: state.ReplicaRole},
		},
	}
	clusterFeatures := structs.ClusterFeatures{
		NodeCount: 2,
		CellGUIDs: []string{"cell1", "cell2"},
	}
	clusterModel := state.NewClusterModel(&state.StateEtcd{}, clusterState)
	plan, err := scheduler.newPlan(clusterModel, clusterFeatures)
	if err != nil {
		t.Fatalf("scheduler.newPlan error: %v", err)
	}
	expectedStepTypes := []string{"AddNode", "WaitForAllMembers", "RemoveLeader(a)", "WaitForAllMembers", "WaitForLeader"}
	stepTypes := plan.stepTypes()
	if !reflect.DeepEqual(stepTypes, expectedStepTypes) {
		t.Fatalf("plan should have steps %v, got %v", expectedStepTypes, stepTypes)
	}
}
// Recreate service instance; invoked via Provision endpoint
func (bkr *Broker) Recreate(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) {
	logger := bkr.newLoggingSession("recreate", lager.Data{})
	defer logger.Info("stop")

	features, err := structs.ClusterFeaturesFromParameters(details.Parameters)
	if err != nil {
		logger.Error("cluster-features", err)
		return resp, false, err
	}

	if err = bkr.assertRecreatePrecondition(instanceID, features); err != nil {
		logger.Error("preconditions.error", err)
		return resp, false, err
	}

	recreationData, err := bkr.callbacks.RestoreRecreationData(instanceID)
	if err != nil {
		err = fmt.Errorf("Cannot recreate service from backup; unable to restore original service instance data: %s", err)
		return
	}

	clusterState := bkr.initClusterStateFromRecreationData(recreationData)
	clusterModel := state.NewClusterModel(bkr.state, clusterState)

	go func() {
		err := bkr.scheduler.RunCluster(clusterModel, features)
		if err != nil {
			logger.Error("run-cluster", err)
			return
		}

		err = bkr.router.AssignPortToCluster(clusterModel.InstanceID(), clusterModel.AllocatedPort())
		if err != nil {
			logger.Error("assign-port", err)
		}
	}()

	return resp, true, err
}
func (bkr *Broker) provision(instanceID structs.ClusterID, details brokerapi.ProvisionDetails, acceptsIncomplete bool) (resp brokerapi.ProvisioningResponse, async bool, err error) {
	if details.ServiceID == "" && details.PlanID == "" {
		return bkr.Recreate(instanceID, details, acceptsIncomplete)
	}

	logger := bkr.newLoggingSession("provision", lager.Data{"instance-id": instanceID})
	defer logger.Info("done")

	features, err := structs.ClusterFeaturesFromParameters(details.Parameters)
	if err != nil {
		logger.Error("cluster-features", err)
		return resp, false, err
	}

	if err = bkr.assertProvisionPrecondition(instanceID, features); err != nil {
		logger.Error("preconditions.error", err)
		return resp, false, err
	}

	port, err := bkr.router.AllocatePort()
	clusterState := bkr.initCluster(instanceID, port, details)
	clusterModel := state.NewClusterModel(bkr.state, clusterState)

	clusterModel.SchedulingMessage("Initializing...")

	if bkr.callbacks.Configured() {
		logger.Info("recreation-data.writing")
		bkr.callbacks.WriteRecreationData(clusterState.RecreationData())
		logger.Info("recreation-data.restoring")
		data, err := bkr.callbacks.RestoreRecreationData(instanceID)
		if err != nil {
			logger.Error("recreation-data.save-failure.error", err)
			return resp, false, err
		}
		if !reflect.DeepEqual(clusterState.RecreationData(), data) {
			err = fmt.Errorf("Cluster recreation data was not saved successfully")
			logger.Error("recreation-data.save-failure.deep-equal", err)
			return resp, false, err
		}
		logger.Info("recreation-data.success")
	}

	var existingClusterData *structs.ClusterRecreationData
	if features.CloneFromServiceName != "" {
		if bkr.backups.BaseURI == "" {
			return resp, false, fmt.Errorf("Broker missing configuration backups.base_uri to support 'clone-from' feature")
		}

		// Confirm that backup can be found before continuing asynchronously
		logger.Info("lookup-service-name.start")
		existingClusterData, err = bkr.lookupClusterDataBackupByServiceInstanceName(details.SpaceGUID, features.CloneFromServiceName, logger)
		if err != nil {
			logger.Error("lookup-service-name.error", err)
			return resp, false, err
		}
		logger.Info("lookup-service-name.success")
	}

	// Continue processing in background
	go func() {
		logger.Info("async-begin")
		defer logger.Info("async-complete")

		if existingClusterData != nil {
			clusterModel.SchedulingMessage(fmt.Sprintf("Cloning existing database %s", existingClusterData.ServiceInstanceName))
			if err := bkr.prepopulateDatabaseFromExistingClusterData(existingClusterData, instanceID, clusterModel, logger); err != nil {
				logger.Error("pre-populate-cluster", err)
				clusterModel.SchedulingError(fmt.Errorf("Unsuccessful pre-populating database from backup. Please contact administrator: %s", err.Error()))
				return
			}
		}

		if err := bkr.scheduler.RunCluster(clusterModel, features); err != nil {
			logger.Error("run-cluster", err)
			return
		}

		if err := bkr.router.AssignPortToCluster(instanceID, port); err != nil {
			logger.Error("assign-port", err)
			clusterModel.SchedulingError(fmt.Errorf("Unsuccessful mapping database to routing mesh. Please contact administrator: %s", err.Error()))
			return
		}

		bkr.fetchAndBackupServiceInstanceName(instanceID, &clusterState, logger)
	}()
	return resp, true, err
}