func (s *Server) ListJobInputs(pipelineDB db.PipelineDB) http.Handler { logger := s.logger.Session("list-job-inputs") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { jobName := r.FormValue(":job_name") pipelineConfig, _, found, err := pipelineDB.GetConfig() if err != nil { logger.Error("failed-to-get-config", err) w.WriteHeader(http.StatusInternalServerError) return } if !found { w.WriteHeader(http.StatusNotFound) return } jobConfig, found := pipelineConfig.Jobs.Lookup(jobName) if !found { w.WriteHeader(http.StatusNotFound) return } versionsDB, err := pipelineDB.LoadVersionsDB() if err != nil { logger.Error("failed-to-load-version-db", err) w.WriteHeader(http.StatusInternalServerError) return } jobInputs := config.JobInputs(jobConfig) inputVersions, found, err := pipelineDB.GetNextInputVersions(versionsDB, jobName, jobInputs) if err != nil { logger.Error("failed-to-get-latest-input-versions", err) w.WriteHeader(http.StatusInternalServerError) return } if !found { w.WriteHeader(http.StatusNotFound) return } buildInputs := make([]atc.BuildInput, len(inputVersions)) for i, input := range inputVersions { resource, _ := pipelineConfig.Resources.Lookup(input.Resource) var config config.JobInput for _, jobInput := range jobInputs { if jobInput.Name == input.Name { config = jobInput break } } buildInputs[i] = present.BuildInput(input, config, resource.Source) } json.NewEncoder(w).Encode(buildInputs) }) }
func (s *Scheduler) BuildLatestInputs(logger lager.Logger, versions *algorithm.VersionsDB, job atc.JobConfig, resources atc.ResourceConfigs, resourceTypes atc.ResourceTypes) error { logger = logger.Session("build-latest") inputs := config.JobInputs(job) if len(inputs) == 0 { // no inputs; no-op return nil } latestInputs, found, err := s.PipelineDB.GetLatestInputVersions(versions, job.Name, inputs) if err != nil { logger.Error("failed-to-get-latest-input-versions", err) return err } if !found { logger.Debug("no-input-versions-available") return nil } checkInputs := []db.BuildInput{} for _, input := range latestInputs { for _, ji := range inputs { if ji.Name == input.Name { if ji.Trigger { checkInputs = append(checkInputs, input) } break } } } if len(checkInputs) == 0 { logger.Debug("no-triggered-input-versions") return nil } existingBuild, found, err := s.PipelineDB.GetJobBuildForInputs(job.Name, checkInputs) if err != nil { logger.Error("could-not-determine-if-inputs-are-already-used", err) return err } if found { logger.Debug("build-already-exists-for-inputs", lager.Data{ "existing-build": existingBuild.ID, }) return nil } build, created, err := s.PipelineDB.CreateJobBuildForCandidateInputs(job.Name) if err != nil { logger.Error("failed-to-create-build", err) return err } if !created { logger.Debug("waiting-for-existing-build-to-determine-inputs", lager.Data{ "existing-build": build.ID, }) return nil } logger.Debug("created-build", lager.Data{"build": build.ID}) jobService, err := NewJobService(job, s.PipelineDB, s.Scanner) if err != nil { logger.Error("failed-to-get-job-service", err) return nil } // NOTE: this is intentionally serial within a scheduler tick, so that // multiple ATCs don't do redundant work to determine a build's inputs. s.ScheduleAndResumePendingBuild(logger, versions, build, job, resources, resourceTypes, jobService) return nil }
Version: db.Version{"some": "other-version"}, PipelineName: "some-pipeline", }, }, }, true, nil) }) It("returns 200 OK", func() { Expect(response.StatusCode).To(Equal(http.StatusOK)) }) It("determined the inputs with the correct versions DB, job name, and inputs", func() { receivedVersionsDB, receivedJob, receivedInputs := pipelineDB.GetLatestInputVersionsArgsForCall(0) Expect(receivedVersionsDB).To(Equal(versionsDB)) Expect(receivedJob).To(Equal("some-job")) Expect(receivedInputs).To(Equal(config.JobInputs(someJob))) }) It("returns the inputs", func() { body, err := ioutil.ReadAll(response.Body) Expect(err).NotTo(HaveOccurred()) Expect(body).To(MatchJSON(`[ { "name": "some-input", "resource": "some-resource", "type": "some-type", "source": {"some": "source"}, "version": {"some": "version"}, "params": {"some": "params"} },
func Job(dbJob db.SavedJob, job atc.JobConfig, groups atc.GroupConfigs, finishedBuild, nextBuild *db.Build) atc.Job { generator := rata.NewRequestGenerator("", web.Routes) req, err := generator.CreateRequest( web.GetJob, rata.Params{"job": job.Name, "pipeline_name": dbJob.PipelineName}, nil, ) if err != nil { panic("failed to generate url: " + err.Error()) } var presentedNextBuild, presentedFinishedBuild *atc.Build if nextBuild != nil { presented := Build(*nextBuild) presentedNextBuild = &presented } if finishedBuild != nil { presented := Build(*finishedBuild) presentedFinishedBuild = &presented } groupNames := []string{} for _, group := range groups { for _, name := range group.Jobs { if name == job.Name { groupNames = append(groupNames, group.Name) } } } sanitizedInputs := []atc.JobInput{} for _, input := range config.JobInputs(job) { sanitizedInputs = append(sanitizedInputs, atc.JobInput{ Name: input.Name, Resource: input.Resource, Passed: input.Passed, Trigger: input.Trigger, }) } sanitizedOutputs := []atc.JobOutput{} for _, output := range config.JobOutputs(job) { sanitizedOutputs = append(sanitizedOutputs, atc.JobOutput{ Name: output.Name, Resource: output.Resource, }) } return atc.Job{ Name: job.Name, URL: req.URL.String(), Paused: dbJob.Paused, FinishedBuild: presentedFinishedBuild, NextBuild: presentedNextBuild, Inputs: sanitizedInputs, Outputs: sanitizedOutputs, Groups: groupNames, } }
) var _ = Describe("Job config", func() { Describe("JobInputs", func() { var ( jobConfig atc.JobConfig inputs []config.JobInput ) BeforeEach(func() { jobConfig = atc.JobConfig{} }) JustBeforeEach(func() { inputs = config.JobInputs(jobConfig) }) Context("with a build plan", func() { Context("with an empty plan", func() { BeforeEach(func() { jobConfig.Plan = atc.PlanSequence{} }) It("returns an empty set of inputs", func() { Expect(inputs).To(BeEmpty()) }) }) Context("with two serial gets", func() { BeforeEach(func() {
func (s *Scheduler) scheduleAndResumePendingBuild(logger lager.Logger, versions *algorithm.VersionsDB, build db.Build, job atc.JobConfig, resources atc.ResourceConfigs) engine.Build { lease, acquired, err := s.BuildsDB.LeaseBuildScheduling(build.ID, 10*time.Second) if err != nil { logger.Error("failed-to-get-lease", err) return nil } if !acquired { return nil } defer lease.Break() logger = logger.WithData(lager.Data{"build": build.ID}) scheduled, err := s.PipelineDB.ScheduleBuild(build.ID, job) if err != nil { logger.Error("failed-to-schedule-build", err) return nil } if !scheduled { logger.Debug("build-could-not-be-scheduled") return nil } buildInputs := config.JobInputs(job) if versions == nil { for _, input := range buildInputs { scanLog := logger.Session("scan", lager.Data{ "input": input.Name, "resource": input.Resource, }) err := s.Scanner.Scan(scanLog, input.Resource) if err != nil { scanLog.Error("failed-to-scan", err) err := s.BuildsDB.ErrorBuild(build.ID, err) if err != nil { logger.Error("failed-to-mark-build-as-errored", err) } return nil } scanLog.Info("done") } loadStart := time.Now() vLog := logger.Session("loading-versions") vLog.Info("start") versions, err = s.PipelineDB.LoadVersionsDB() if err != nil { vLog.Error("failed", err) return nil } vLog.Info("done", lager.Data{"took": time.Since(loadStart).String()}) } inputs, found, err := s.PipelineDB.GetLatestInputVersions(versions, job.Name, buildInputs) if err != nil { logger.Error("failed-to-get-latest-input-versions", err) return nil } if !found { logger.Debug("no-input-versions-available") return nil } err = s.PipelineDB.UseInputsForBuild(build.ID, inputs) if err != nil { logger.Error("failed-to-use-inputs-for-build", err) return nil } plan := s.Factory.Create(job, resources, inputs) createdBuild, err := s.Engine.CreateBuild(logger, build, plan) if err != nil { logger.Error("failed-to-create-build", err) return nil } if createdBuild != nil { logger.Info("building") go createdBuild.Resume(logger) } return createdBuild }
func (s jobService) getBuildInputs(logger lager.Logger, build db.Build, buildPrep db.BuildPreparation, versions *algorithm.VersionsDB) ([]db.BuildInput, db.BuildPreparation, string, error) { buildInputs := config.JobInputs(s.JobConfig) if versions == nil { for _, input := range buildInputs { buildPrep.Inputs[input.Name] = db.BuildPreparationStatusUnknown } err := s.DB.UpdateBuildPreparation(buildPrep) if err != nil { return nil, buildPrep, "failed-to-update-build-prep-with-inputs", err } for _, input := range buildInputs { scanLog := logger.Session("scan", lager.Data{ "input": input.Name, "resource": input.Resource, }) buildPrep = s.cloneBuildPrep(buildPrep) buildPrep.Inputs[input.Name] = db.BuildPreparationStatusBlocking err := s.DB.UpdateBuildPreparation(buildPrep) if err != nil { return nil, buildPrep, "failed-to-update-build-prep-with-blocking-input", err } err = s.Scanner.Scan(scanLog, input.Resource) if err != nil { return nil, buildPrep, "failed-to-scan", err } buildPrep = s.cloneBuildPrep(buildPrep) buildPrep.Inputs[input.Name] = db.BuildPreparationStatusNotBlocking err = s.DB.UpdateBuildPreparation(buildPrep) if err != nil { return nil, buildPrep, "failed-to-update-build-prep-with-not-blocking-input", err } scanLog.Info("done") } loadStart := time.Now() vLog := logger.Session("loading-versions") vLog.Info("start") versions, err = s.DB.LoadVersionsDB() if err != nil { vLog.Error("failed", err) return nil, buildPrep, "failed-to-load-versions-db", err } vLog.Info("done", lager.Data{"took": time.Since(loadStart).String()}) } else { for _, input := range buildInputs { buildPrep.Inputs[input.Name] = db.BuildPreparationStatusNotBlocking } err := s.DB.UpdateBuildPreparation(buildPrep) if err != nil { return nil, buildPrep, "failed-to-update-build-prep-with-discovered-inputs", err } } buildPrep.InputsSatisfied = db.BuildPreparationStatusBlocking err := s.DB.UpdateBuildPreparation(buildPrep) if err != nil { return nil, buildPrep, "failed-to-update-build-prep-with-discovered-inputs", err } inputs, message, err := s.determineInputs(versions, buildInputs, build) if err != nil || message != "" { return nil, buildPrep, message, err } buildPrep.InputsSatisfied = db.BuildPreparationStatusNotBlocking err = s.DB.UpdateBuildPreparation(buildPrep) if err != nil { return nil, buildPrep, "failed-to-update-build-prep-with-inputs-satisfied", err } return inputs, buildPrep, "", nil }