func (f *Formation) jobConfig(name string, hostID string) *host.Job {
return utils.JobConfig(&ct.ExpandedFormation{
App: &ct.App{ID: f.AppID, Name: f.AppName},
Release: f.Release,
Artifact: f.Artifact,
}, name, hostID)
}
func (f *Formation) jobConfig(name string) (*host.Job, error) {
return utils.JobConfig(&ct.ExpandedFormation{
App: &ct.App{ID: f.AppID},
Release: f.Release,
Artifact: f.Artifact,
}, name)
}
func (f *ClusterFixer) StartScheduler(client *controller.Client, cf *ct.Formation) error {
if _, err := discoverd.NewService("controller-scheduler").Leader(); err != nil && !discoverd.IsNotFound(err) {
return fmt.Errorf("error getting scheduler leader: %s", err)
} else if err == nil {
f.l.Info("scheduler looks up, moving on")
return nil
}
f.l.Info("scheduler is not up, attempting to fix")
// start scheduler
ef, err := utils.ExpandFormation(client, cf)
if err != nil {
return err
}
schedulerJob := utils.JobConfig(ef, "scheduler", f.hosts[0].ID(), "")
if err := f.hosts[0].AddJob(schedulerJob); err != nil {
return fmt.Errorf("error starting scheduler job on %s: %s", f.hosts[0].ID(), err)
}
f.l.Info("started scheduler job")
return nil
}
func (a *RunAppAction) Run(s *State) error {
if a.AppStep != "" {
data, err := getAppStep(s, a.AppStep)
if err != nil {
return err
}
a.App = data.App
procs := a.Processes
a.ExpandedFormation = data.ExpandedFormation
a.Processes = procs
}
as := &RunAppState{
ExpandedFormation: a.ExpandedFormation,
Resources: make([]*resource.Resource, 0, len(a.Resources)),
Providers: make([]*ct.Provider, 0, len(a.Resources)),
}
s.StepData[a.ID] = as
if a.App == nil {
a.App = &ct.App{}
}
if a.App.ID == "" {
a.App.ID = random.UUID()
}
if a.Artifact == nil {
return errors.New("bootstrap: artifact must be set")
}
if a.Artifact.ID == "" {
a.Artifact.ID = random.UUID()
}
if a.Release == nil {
return errors.New("bootstrap: release must be set")
}
if a.Release.ID == "" {
a.Release.ID = random.UUID()
}
a.Release.ArtifactID = a.Artifact.ID
if a.Release.Env == nil {
a.Release.Env = make(map[string]string)
}
interpolateRelease(s, a.Release)
for _, p := range a.Resources {
server, err := resource.NewServer(p.URL)
if err != nil {
return err
}
res, err := server.Provision(nil)
server.Close()
if err != nil {
return err
}
as.Providers = append(as.Providers, p)
as.Resources = append(as.Resources, res)
for k, v := range res.Env {
a.Release.Env[k] = v
}
}
cc, err := s.ClusterClient()
if err != nil {
return err
}
hosts, err := cc.ListHosts()
if err != nil {
return err
}
hostIDs := make([]string, 0, len(hosts))
for id := range hosts {
hostIDs = append(hostIDs, id)
}
for typ, count := range a.Processes {
for i := 0; i < count; i++ {
job, err := startJob(s, hostIDs[i%len(hosts)], utils.JobConfig(a.ExpandedFormation, typ))
if err != nil {
return err
}
as.Jobs = append(as.Jobs, *job)
}
}
return nil
}
func (a *RunAppAction) Run(s *State) error {
if a.AppStep != "" {
data, err := getAppStep(s, a.AppStep)
if err != nil {
return err
}
a.App = data.App
procs := a.Processes
a.ExpandedFormation = data.ExpandedFormation
a.Processes = procs
}
as := &RunAppState{
ExpandedFormation: a.ExpandedFormation,
Resources: make([]*resource.Resource, 0, len(a.Resources)),
Providers: make([]*ct.Provider, 0, len(a.Resources)),
}
s.StepData[a.ID] = as
if a.App == nil {
a.App = &ct.App{}
}
if a.App.ID == "" {
a.App.ID = random.UUID()
}
if a.ImageArtifact == nil {
return errors.New("bootstrap: artifact must be set")
}
if a.ImageArtifact.ID == "" {
a.ImageArtifact.ID = random.UUID()
}
if a.Release == nil {
return errors.New("bootstrap: release must be set")
}
if a.Release.ID == "" {
a.Release.ID = random.UUID()
}
a.Release.ArtifactIDs = []string{a.ImageArtifact.ID}
if a.Release.Env == nil {
a.Release.Env = make(map[string]string)
}
interpolateRelease(s, a.Release)
for _, p := range a.Resources {
u, err := url.Parse(p.URL)
if err != nil {
return err
}
lookupDiscoverdURLHost(s, u, time.Second)
res, err := resource.Provision(u.String(), nil)
if err != nil {
return err
}
as.Providers = append(as.Providers, p)
as.Resources = append(as.Resources, res)
for k, v := range res.Env {
a.Release.Env[k] = v
}
}
for typ, count := range a.Processes {
if s.Singleton && count > 1 {
a.Processes[typ] = 1
count = 1
}
hosts := s.ShuffledHosts()
if a.ExpandedFormation.Release.Processes[typ].Omni {
count = len(hosts)
}
for i := 0; i < count; i++ {
host := hosts[i%len(hosts)]
config := utils.JobConfig(a.ExpandedFormation, typ, host.ID(), "")
hostresource.SetDefaults(&config.Resources)
if a.ExpandedFormation.Release.Processes[typ].Data {
if err := utils.ProvisionVolume(host, config); err != nil {
return err
}
}
if err := startJob(s, host, config); err != nil {
return err
}
}
}
return nil
}
func (a *RunAppAction) Run(s *State) error {
if a.AppStep != "" {
data, err := getAppStep(s, a.AppStep)
if err != nil {
return err
}
a.App = data.App
procs := a.Processes
a.ExpandedFormation = data.ExpandedFormation
a.Processes = procs
}
as := &RunAppState{
ExpandedFormation: a.ExpandedFormation,
Resources: make([]*resource.Resource, 0, len(a.Resources)),
Providers: make([]*ct.Provider, 0, len(a.Resources)),
}
s.StepData[a.ID] = as
if a.App == nil {
a.App = &ct.App{}
}
if a.App.ID == "" {
a.App.ID = random.UUID()
}
if a.Artifact == nil {
return errors.New("bootstrap: artifact must be set")
}
if a.Artifact.ID == "" {
a.Artifact.ID = random.UUID()
}
if a.Release == nil {
return errors.New("bootstrap: release must be set")
}
if a.Release.ID == "" {
a.Release.ID = random.UUID()
}
a.Release.ArtifactID = a.Artifact.ID
if a.Release.Env == nil {
a.Release.Env = make(map[string]string)
}
interpolateRelease(s, a.Release)
for _, p := range a.Resources {
u, err := url.Parse(p.URL)
if err != nil {
return err
}
lookupDiscoverdURLHost(u, time.Second)
res, err := resource.Provision(u.String(), nil)
if err != nil {
return err
}
as.Providers = append(as.Providers, p)
as.Resources = append(as.Resources, res)
for k, v := range res.Env {
a.Release.Env[k] = v
}
}
cc, err := s.ClusterClient()
if err != nil {
return err
}
for typ, count := range a.Processes {
if s.Singleton && count > 1 {
a.Processes[typ] = 1
count = 1
}
hosts, err := cc.ListHosts()
if err != nil {
return err
}
sort.Sort(schedutil.HostSlice(hosts))
for i := 0; i < count; i++ {
hostID := hosts[i%len(hosts)].ID
config := utils.JobConfig(a.ExpandedFormation, typ, hostID)
if a.ExpandedFormation.Release.Processes[typ].Data {
if err := utils.ProvisionVolume(cc, hostID, config); err != nil {
return err
}
}
job, err := startJob(s, hostID, config)
if err != nil {
return err
}
as.Jobs = append(as.Jobs, *job)
}
}
return nil
}
func jobConfig(job *Job, hostID string) *host.Job {
return utils.JobConfig(job.Formation.ExpandedFormation, job.Type, hostID, job.ID)
}
func (f *clusterFixer) fixController(instances []*discoverd.Instance, startScheduler bool) error {
f.l.Info("found controller instance, checking critical formations")
inst := instances[0]
client, err := controller.NewClient("http://"+inst.Addr, inst.Meta["AUTH_KEY"])
if err != nil {
return fmt.Errorf("unexpected error creating controller client: %s", err)
}
// check that formations for critical components are expected
apps := []string{"controller", "router", "discoverd", "flannel", "postgres"}
changes := make(map[string]*ct.Formation, len(apps))
var controllerFormation *ct.Formation
for _, app := range apps {
release, err := client.GetAppRelease(app)
if err != nil {
return fmt.Errorf("error getting %s release: %s", app, err)
}
formation, err := client.GetFormation(app, release.ID)
if err != nil {
// TODO: handle ErrNotFound
return fmt.Errorf("error getting %s formation: %s", app, err)
}
if app == "controller" {
controllerFormation = formation
}
for typ := range release.Processes {
var want int
if app == "postgres" && typ == "postgres" && len(f.hosts) > 1 && formation.Processes[typ] < 3 {
want = 3
} else if formation.Processes[typ] < 1 {
want = 1
}
if want > 0 {
f.l.Info("found broken formation", "app", app, "process", typ)
if _, ok := changes[app]; !ok {
if formation.Processes == nil {
formation.Processes = make(map[string]int)
}
changes[app] = formation
}
changes[app].Processes[typ] = want
}
}
}
for app, formation := range changes {
f.l.Info("fixing broken formation", "app", app)
if err := client.PutFormation(formation); err != nil {
return fmt.Errorf("error putting %s formation: %s", app, err)
}
}
if startScheduler {
if _, err := discoverd.NewService("controller-scheduler").Leader(); err != nil && !discoverd.IsNotFound(err) {
return fmt.Errorf("error getting scheduler leader: %s", err)
} else if err == nil {
f.l.Info("scheduler looks up, moving on")
return nil
}
f.l.Info("scheduler is not up, attempting to fix")
// start scheduler
ef, err := utils.ExpandFormation(client, controllerFormation)
if err != nil {
return err
}
schedulerJob := utils.JobConfig(ef, "scheduler", f.hosts[0].ID())
if err := f.hosts[0].AddJob(schedulerJob); err != nil {
return fmt.Errorf("error starting scheduler job on %s: %s", f.hosts[0].ID(), err)
}
f.l.Info("started scheduler job")
}
return nil
}
func jobConfig(req *JobRequest, hostID string) *host.Job {
return utils.JobConfig(req.Job.Formation.ExpandedFormation, req.Type, hostID)
}