func activateAndRunSteps(workflow models.WorkflowModel, defaultStepLibSource string, buildRunResults models.BuildRunResultsModel, environments *[]envmanModels.EnvironmentItemModel, isLastWorkflow bool) models.BuildRunResultsModel { log.Debugln("[BITRISE_CLI] - Activating and running steps") // ------------------------------------------ // In function global variables - These are global for easy use in local register step run result methods. var stepStartTime time.Time // ------------------------------------------ // In function method - Registration methods, for register step run results. registerStepRunResults := func(step stepmanModels.StepModel, stepInfoPtr stepmanModels.StepInfoModel, stepIdxPtr int, runIf string, resultCode, exitCode int, err error, isLastStep, printStepHeader bool) { if printStepHeader { bitrise.PrintRunningStepHeader(stepInfoPtr, step, stepIdxPtr) } stepInfoCopy := stepmanModels.StepInfoModel{ ID: stepInfoPtr.ID, Title: stepInfoPtr.Title, Version: stepInfoPtr.Version, Latest: stepInfoPtr.Latest, SupportURL: stepInfoPtr.SupportURL, SourceCodeURL: stepInfoPtr.SourceCodeURL, GlobalInfo: stepInfoPtr.GlobalInfo, } stepResults := models.StepRunResultsModel{ StepInfo: stepInfoCopy, Status: resultCode, Idx: buildRunResults.ResultsCount(), RunTime: time.Now().Sub(stepStartTime), Error: err, ExitCode: exitCode, } isExitStatusError := true if err != nil { isExitStatusError = errorutil.IsExitStatusError(err) } switch resultCode { case models.StepRunStatusCodeSuccess: buildRunResults.SuccessSteps = append(buildRunResults.SuccessSteps, stepResults) break case models.StepRunStatusCodeFailed: if !isExitStatusError { log.Errorf("Step (%s) failed, error: %s", stepInfoCopy.Title, err) } buildRunResults.FailedSteps = append(buildRunResults.FailedSteps, stepResults) break case models.StepRunStatusCodeFailedSkippable: if !isExitStatusError { log.Warnf("Step (%s) failed, but was marked as skippable, error: %s", stepInfoCopy.Title, err) } else { log.Warnf("Step (%s) failed, but was marked as skippable", stepInfoCopy.Title) } buildRunResults.FailedSkippableSteps = append(buildRunResults.FailedSkippableSteps, stepResults) break case models.StepRunStatusCodeSkipped: log.Warnf("A previous step failed, and this step (%s) was not marked as IsAlwaysRun, skipped", stepInfoCopy.Title) buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break case models.StepRunStatusCodeSkippedWithRunIf: log.Warn("The step's (" + stepInfoCopy.Title + ") Run-If expression evaluated to false - skipping") if runIf != "" { log.Info("The Run-If expression was: ", colorstring.Blue(runIf)) } buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break default: log.Error("Unkown result code") return } bitrise.PrintRunningStepFooter(stepResults, isLastStep) } // ------------------------------------------ // Main - Preparing & running the steps for idx, stepListItm := range workflow.Steps { // Per step variables stepStartTime = time.Now() isLastStep := isLastWorkflow && (idx == len(workflow.Steps)-1) stepInfoPtr := stepmanModels.StepInfoModel{} stepIdxPtr := idx // Per step cleanup if err := bitrise.SetBuildFailedEnv(buildRunResults.IsBuildFailed()); err != nil { log.Error("Failed to set Build Status envs") } if err := bitrise.CleanupStepWorkDir(); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } // // Preparing the step if err := tools.EnvmanInitAtPath(configs.InputEnvstorePath); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } if err := bitrise.ExportEnvironmentsList(*environments); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } // Get step id & version data compositeStepIDStr, workflowStep, err := models.GetStepIDStepDataPair(stepListItm) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } stepInfoPtr.ID = compositeStepIDStr if workflowStep.Title != nil && *workflowStep.Title != "" { stepInfoPtr.Title = *workflowStep.Title } else { stepInfoPtr.Title = compositeStepIDStr } stepIDData, err := models.CreateStepIDDataFromString(compositeStepIDStr, defaultStepLibSource) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } stepInfoPtr.ID = stepIDData.IDorURI if stepInfoPtr.Title == "" { stepInfoPtr.Title = stepIDData.IDorURI } stepInfoPtr.Version = stepIDData.Version stepInfoPtr.StepLib = stepIDData.SteplibSource // // Activating the step stepDir := configs.BitriseWorkStepsDirPath stepYMLPth := filepath.Join(configs.BitriseWorkDirPath, "current_step.yml") if stepIDData.SteplibSource == "path" { log.Debugf("[BITRISE_CLI] - Local step found: (path:%s)", stepIDData.IDorURI) stepAbsLocalPth, err := pathutil.AbsPath(stepIDData.IDorURI) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } log.Debugln("stepAbsLocalPth:", stepAbsLocalPth, "|stepDir:", stepDir) if err := cmdex.CopyDir(stepAbsLocalPth, stepDir, true); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } if err := cmdex.CopyFile(filepath.Join(stepAbsLocalPth, "step.yml"), stepYMLPth); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } } else if stepIDData.SteplibSource == "git" { log.Debugf("[BITRISE_CLI] - Remote step, with direct git uri: (uri:%s) (tag-or-branch:%s)", stepIDData.IDorURI, stepIDData.Version) if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, stepDir, stepIDData.Version); err != nil { if strings.HasPrefix(stepIDData.IDorURI, "git@") { fmt.Println(colorstring.Yellow(`Note: if the step's repository is an open source one,`)) fmt.Println(colorstring.Yellow(`you should probably use a "https://..." git clone URL,`)) fmt.Println(colorstring.Yellow(`instead of the "git@..." git clone URL which usually requires authentication`)) fmt.Println(colorstring.Yellow(`even if the repository is open source!`)) } registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } if err := cmdex.CopyFile(filepath.Join(stepDir, "step.yml"), stepYMLPth); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } } else if stepIDData.SteplibSource == "_" { log.Debugf("[BITRISE_CLI] - Steplib independent step, with direct git uri: (uri:%s) (tag-or-branch:%s)", stepIDData.IDorURI, stepIDData.Version) // Steplib independent steps are completly defined in workflow stepYMLPth = "" if err := workflowStep.FillMissingDefaults(); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, stepDir, stepIDData.Version); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } } else if stepIDData.SteplibSource != "" { log.Debugf("[BITRISE_CLI] - Steplib (%s) step (id:%s) (version:%s) found, activating step", stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err := tools.StepmanSetup(stepIDData.SteplibSource); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } isLatestVersionOfStep := (stepIDData.Version == "") if isLatestVersionOfStep && !buildRunResults.IsStepLibUpdated(stepIDData.SteplibSource) { log.Infof("Step uses latest version -- Updating StepLib ...") if err := tools.StepmanUpdate(stepIDData.SteplibSource); err != nil { log.Warnf("Step uses latest version, but failed to update StepLib, err: %s", err) } else { buildRunResults.StepmanUpdates[stepIDData.SteplibSource]++ } } outStr, err := tools.StepmanJSONStepLibStepInfo(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err != nil { if buildRunResults.IsStepLibUpdated(stepIDData.SteplibSource) { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("StepmanJSONStepLibStepInfo failed, err: %s", err), isLastStep, true) continue } // May StepLib should be updated log.Infof("Step info not found in StepLib (%s) -- Updating ...", stepIDData.SteplibSource) if err := tools.StepmanUpdate(stepIDData.SteplibSource); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } buildRunResults.StepmanUpdates[stepIDData.SteplibSource]++ outStr, err = tools.StepmanJSONStepLibStepInfo(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("StepmanJSONStepLibStepInfo failed, err: %s", err), isLastStep, true) continue } } stepInfo, err := stepmanModels.StepInfoModel{}.CreateFromJSON(outStr) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("CreateFromJSON failed, err: %s", err), isLastStep, true) continue } stepInfoPtr.ID = stepInfo.ID if stepInfoPtr.Title == "" { stepInfoPtr.Title = stepInfo.ID } stepInfoPtr.Version = stepInfo.Version stepInfoPtr.Latest = stepInfo.Latest stepInfoPtr.GlobalInfo = stepInfo.GlobalInfo if err := tools.StepmanActivate(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version, stepDir, stepYMLPth); err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } else { log.Debugf("[BITRISE_CLI] - Step activated: (ID:%s) (version:%s)", stepIDData.IDorURI, stepIDData.Version) } } else { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, fmt.Errorf("Invalid stepIDData: No SteplibSource or LocalPath defined (%v)", stepIDData), isLastStep, true) continue } // Fill step info with default step info, if exist mergedStep := workflowStep if stepYMLPth != "" { specStep, err := bitrise.ReadSpecStep(stepYMLPth) log.Debugf("Spec read from YML: %#v\n", specStep) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } mergedStep, err = models.MergeStepWith(specStep, workflowStep) if err != nil { registerStepRunResults(stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, "", models.StepRunStatusCodeFailed, 1, err, isLastStep, true) continue } } if mergedStep.SupportURL != nil { stepInfoPtr.SupportURL = *mergedStep.SupportURL } if mergedStep.SourceCodeURL != nil { stepInfoPtr.SourceCodeURL = *mergedStep.SourceCodeURL } // // Run step bitrise.PrintRunningStepHeader(stepInfoPtr, mergedStep, idx) if mergedStep.RunIf != nil && *mergedStep.RunIf != "" { outStr, err := tools.EnvmanJSONPrint(configs.InputEnvstorePath) if err != nil { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, fmt.Errorf("EnvmanJSONPrint failed, err: %s", err), isLastStep, false) continue } envList, err := envmanModels.NewEnvJSONList(outStr) if err != nil { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, fmt.Errorf("CreateFromJSON failed, err: %s", err), isLastStep, false) continue } isRun, err := bitrise.EvaluateTemplateToBool(*mergedStep.RunIf, configs.IsCIMode, configs.IsPullRequestMode, buildRunResults, envList) if err != nil { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, err, isLastStep, false) continue } if !isRun { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeSkippedWithRunIf, 0, err, isLastStep, false) continue } } isAlwaysRun := stepmanModels.DefaultIsAlwaysRun if mergedStep.IsAlwaysRun != nil { isAlwaysRun = *mergedStep.IsAlwaysRun } else { log.Warn("Step (%s) mergedStep.IsAlwaysRun is nil, should not!", stepIDData.IDorURI) } if buildRunResults.IsBuildFailed() && !isAlwaysRun { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeSkipped, 0, err, isLastStep, false) } else { exit, outEnvironments, err := runStep(mergedStep, stepIDData, stepDir, *environments, buildRunResults) *environments = append(*environments, outEnvironments...) if err != nil { if *mergedStep.IsSkippable { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeFailedSkippable, exit, err, isLastStep, false) } else { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeFailed, exit, err, isLastStep, false) } } else { registerStepRunResults(mergedStep, stepInfoPtr, stepIdxPtr, *mergedStep.RunIf, models.StepRunStatusCodeSuccess, 0, nil, isLastStep, false) } } } return buildRunResults }
func removeStepDefaultsAndFillStepOutputs(stepListItem *models.StepListItemModel, defaultStepLibSource string) error { // Create stepIDData compositeStepIDStr, workflowStep, err := models.GetStepIDStepDataPair(*stepListItem) if err != nil { return err } stepIDData, err := models.CreateStepIDDataFromString(compositeStepIDStr, defaultStepLibSource) if err != nil { return err } // Activate step - get step.yml tempStepCloneDirPath, err := pathutil.NormalizedOSTempDirPath("step_clone") if err != nil { return err } tempStepYMLDirPath, err := pathutil.NormalizedOSTempDirPath("step_yml") if err != nil { return err } tempStepYMLFilePath := filepath.Join(tempStepYMLDirPath, "step.yml") if stepIDData.SteplibSource == "path" { stepAbsLocalPth, err := pathutil.AbsPath(stepIDData.IDorURI) if err != nil { return err } if err := cmdex.CopyFile(filepath.Join(stepAbsLocalPth, "step.yml"), tempStepYMLFilePath); err != nil { return err } } else if stepIDData.SteplibSource == "git" { if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, tempStepCloneDirPath, stepIDData.Version); err != nil { return err } if err := cmdex.CopyFile(filepath.Join(tempStepCloneDirPath, "step.yml"), tempStepYMLFilePath); err != nil { return err } } else if stepIDData.SteplibSource == "_" { // Steplib independent steps are completly defined in workflow tempStepYMLFilePath = "" } else if stepIDData.SteplibSource != "" { if err := tools.StepmanSetup(stepIDData.SteplibSource); err != nil { return err } if err := tools.StepmanActivate(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version, tempStepCloneDirPath, tempStepYMLFilePath); err != nil { return err } } else { return errors.New("Failed to fill step ouputs: unkown SteplibSource") } // Fill outputs if tempStepYMLFilePath != "" { specStep, err := ReadSpecStep(tempStepYMLFilePath) if err != nil { return err } if workflowStep.Title != nil && specStep.Title != nil && *workflowStep.Title == *specStep.Title { workflowStep.Title = nil } if workflowStep.Description != nil && specStep.Description != nil && *workflowStep.Description == *specStep.Description { workflowStep.Description = nil } if workflowStep.Summary != nil && specStep.Summary != nil && *workflowStep.Summary == *specStep.Summary { workflowStep.Summary = nil } if workflowStep.Website != nil && specStep.Website != nil && *workflowStep.Website == *specStep.Website { workflowStep.Website = nil } if workflowStep.SourceCodeURL != nil && specStep.SourceCodeURL != nil && *workflowStep.SourceCodeURL == *specStep.SourceCodeURL { workflowStep.SourceCodeURL = nil } if workflowStep.SupportURL != nil && specStep.SupportURL != nil && *workflowStep.SupportURL == *specStep.SupportURL { workflowStep.SupportURL = nil } workflowStep.PublishedAt = nil if workflowStep.Source != nil && specStep.Source != nil { if workflowStep.Source.Git == specStep.Source.Git { workflowStep.Source.Git = "" } if workflowStep.Source.Commit == specStep.Source.Commit { workflowStep.Source.Commit = "" } } if isStringSliceWithSameElements(workflowStep.HostOsTags, specStep.HostOsTags) { workflowStep.HostOsTags = []string{} } if isStringSliceWithSameElements(workflowStep.ProjectTypeTags, specStep.ProjectTypeTags) { workflowStep.ProjectTypeTags = []string{} } if isStringSliceWithSameElements(workflowStep.TypeTags, specStep.TypeTags) { workflowStep.TypeTags = []string{} } if isDependencySliceWithSameElements(workflowStep.Dependencies, specStep.Dependencies) { workflowStep.Dependencies = []stepmanModels.DependencyModel{} } if workflowStep.IsRequiresAdminUser != nil && specStep.IsRequiresAdminUser != nil && *workflowStep.IsRequiresAdminUser == *specStep.IsRequiresAdminUser { workflowStep.IsRequiresAdminUser = nil } if workflowStep.IsAlwaysRun != nil && specStep.IsAlwaysRun != nil && *workflowStep.IsAlwaysRun == *specStep.IsAlwaysRun { workflowStep.IsAlwaysRun = nil } if workflowStep.IsSkippable != nil && specStep.IsSkippable != nil && *workflowStep.IsSkippable == *specStep.IsSkippable { workflowStep.IsSkippable = nil } if workflowStep.RunIf != nil && specStep.RunIf != nil && *workflowStep.RunIf == *specStep.RunIf { workflowStep.RunIf = nil } inputs := []envmanModels.EnvironmentItemModel{} for _, input := range workflowStep.Inputs { sameValue := false wfKey, wfValue, err := input.GetKeyValuePair() if err != nil { return err } wfOptions, err := input.GetOptions() if err != nil { return err } sInput, err := getInputByKey(specStep.Inputs, wfKey) if err != nil { return err } _, sValue, err := sInput.GetKeyValuePair() if err != nil { return err } if wfValue == sValue { sameValue = true } sOptions, err := sInput.GetOptions() if err != nil { return err } hasOptions := false if wfOptions.Title != nil && sOptions.Title != nil && *wfOptions.Title == *sOptions.Title { wfOptions.Title = nil } else { hasOptions = true } if wfOptions.Description != nil && sOptions.Description != nil && *wfOptions.Description == *sOptions.Description { wfOptions.Description = nil } else { hasOptions = true } if wfOptions.Summary != nil && sOptions.Summary != nil && *wfOptions.Summary == *sOptions.Summary { wfOptions.Summary = nil } else { hasOptions = true } if isStringSliceWithSameElements(wfOptions.ValueOptions, sOptions.ValueOptions) { wfOptions.ValueOptions = []string{} } else { hasOptions = true } if wfOptions.IsRequired != nil && sOptions.IsRequired != nil && *wfOptions.IsRequired == *sOptions.IsRequired { wfOptions.IsRequired = nil } else { hasOptions = true } if wfOptions.IsExpand != nil && sOptions.IsExpand != nil && *wfOptions.IsExpand == *sOptions.IsExpand { wfOptions.IsExpand = nil } else { hasOptions = true } if wfOptions.IsDontChangeValue != nil && sOptions.IsDontChangeValue != nil && *wfOptions.IsDontChangeValue == *sOptions.IsDontChangeValue { wfOptions.IsDontChangeValue = nil } else { hasOptions = true } if !hasOptions && sameValue { // default env } else { if hasOptions { input[envmanModels.OptionsKey] = wfOptions } else { delete(input, envmanModels.OptionsKey) } inputs = append(inputs, input) } } workflowStep.Inputs = inputs // We need only key-value and title from spec outputs outputs := []envmanModels.EnvironmentItemModel{} for _, output := range specStep.Outputs { sKey, sValue, err := output.GetKeyValuePair() if err != nil { return err } sOptions, err := output.GetOptions() if err != nil { return err } newOutput := envmanModels.EnvironmentItemModel{ sKey: sValue, envmanModels.OptionsKey: envmanModels.EnvironmentItemOptionsModel{ Title: sOptions.Title, }, } outputs = append(outputs, newOutput) } workflowStep.Outputs = outputs (*stepListItem)[compositeStepIDStr] = workflowStep } // Cleanup if err := cmdex.RemoveDir(tempStepCloneDirPath); err != nil { return errors.New(fmt.Sprint("Failed to remove step clone dir: ", err)) } if err := cmdex.RemoveDir(tempStepYMLDirPath); err != nil { return errors.New(fmt.Sprint("Failed to remove step clone dir: ", err)) } return nil }
// ConvertOldWorkflow ... func ConvertOldWorkflow(oldWorkflow oldmodels.WorkflowModel) (bitriseModels.WorkflowModel, string, error) { // Step conversion containsCertificateStep := false defaultStepLib := "" stepList := []bitriseModels.StepListItemModel{} for _, oldStep := range oldWorkflow.Steps { oldStepID := oldStep.ID newStep, err := oldStep.Convert() if err != nil { return bitriseModels.WorkflowModel{}, "", err } converterFunc, found := getNewStepIDAndConverter(oldStepID) if found { log.Infof("Convertable step found (%s)", oldStepID) fmt.Println() convertedStepListItems, err := converterFunc(newStep) if err != nil { return bitriseModels.WorkflowModel{}, "", err } for _, stepListItem := range convertedStepListItems { stepID, _, err := bitriseModels.GetStepIDStepDataPair(stepListItem) if err != nil { return bitriseModels.WorkflowModel{}, "", err } if strings.Contains(stepID, utils.CertificateStepID) { if containsCertificateStep { continue } else { containsCertificateStep = true } } defaultStepLib = utils.BitriseVerifiedStepLibGitURI stepList = append(stepList, stepListItem) } } else { log.Infof("Step (%s) not convertable", oldStepID) fmt.Println() if err := newStep.FillMissingDefaults(); err != nil { return bitriseModels.WorkflowModel{}, "", err } _, _, version := oldStep.GetStepLibIDVersionData() stepIDDataString := "_::" + newStep.Source.Git + "@" + version stepListItem := bitriseModels.StepListItemModel{ stepIDDataString: newStep, } stepList = append(stepList, stepListItem) } } // Workflow environments environments, err := oldWorkflow.GetEnvironments() if err != nil { return bitriseModels.WorkflowModel{}, "", err } return bitriseModels.WorkflowModel{ Environments: environments, Steps: stepList, }, defaultStepLib, nil }
func activateAndRunSteps(workflow models.WorkflowModel, defaultStepLibSource string, buildRunResults models.BuildRunResultsModel, environments *[]envmanModels.EnvironmentItemModel, isLastWorkflow bool) models.BuildRunResultsModel { log.Debugln("[BITRISE_CLI] - Activating and running steps") // ------------------------------------------ // In function global variables - These are global for easy use in local register step run result methods. var stepStartTime time.Time // Holds pointer to current step info, for easy usage in local register step run result methods. // The value is filled with the current running step info. var stepInfoPtr models.StepInfoModel // ------------------------------------------ // In function method - Registration methods, for register step run results. registerStepRunResults := func(runIf string, resultCode, exitCode int, err error, isLastStep bool) { stepInfoCopy := models.StepInfoModel{ ID: stepInfoPtr.ID, Version: stepInfoPtr.Version, Latest: stepInfoPtr.Latest, } stepResults := models.StepRunResultsModel{ StepInfo: stepInfoCopy, Status: resultCode, Idx: buildRunResults.ResultsCount(), RunTime: time.Now().Sub(stepStartTime), Error: err, ExitCode: exitCode, } switch resultCode { case models.StepRunStatusCodeSuccess: buildRunResults.SuccessSteps = append(buildRunResults.SuccessSteps, stepResults) break case models.StepRunStatusCodeFailed: log.Errorf("Step (%s) failed, error: (%v)", stepInfoCopy.ID, err) buildRunResults.FailedSteps = append(buildRunResults.FailedSteps, stepResults) break case models.StepRunStatusCodeFailedSkippable: log.Warnf("Step (%s) failed, but was marked as skippable, error: (%v)", stepInfoCopy.ID, err) buildRunResults.FailedSkippableSteps = append(buildRunResults.FailedSkippableSteps, stepResults) break case models.StepRunStatusCodeSkipped: log.Warnf("A previous step failed, and this step (%s) was not marked as IsAlwaysRun, skipped", stepInfoCopy.ID) buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break case models.StepRunStatusCodeSkippedWithRunIf: log.Warn("The step's (" + stepInfoCopy.ID + ") Run-If expression evaluated to false - skipping") if runIf != "" { log.Info("The Run-If expression was: ", colorstring.Blue(runIf)) } buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break default: log.Error("Unkown result code") return } bitrise.PrintStepSummary(stepResults, isLastStep) } // ------------------------------------------ // Main - Preparing & running the steps for idx, stepListItm := range workflow.Steps { // Per step variables stepStartTime = time.Now() isLastStep := isLastWorkflow && (idx == len(workflow.Steps)-1) stepInfoPtr = models.StepInfoModel{} // Per step cleanup if err := bitrise.SetBuildFailedEnv(buildRunResults.IsBuildFailed()); err != nil { log.Error("Failed to set Build Status envs") } if err := bitrise.CleanupStepWorkDir(); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } // // Preparing the step // Get step id & version data compositeStepIDStr, workflowStep, err := models.GetStepIDStepDataPair(stepListItm) stepInfoPtr.ID = compositeStepIDStr if workflowStep.Title != nil && *workflowStep.Title != "" { stepInfoPtr.ID = *workflowStep.Title } if err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepIDData, err := models.CreateStepIDDataFromString(compositeStepIDStr, defaultStepLibSource) stepInfoPtr.ID = stepIDData.IDorURI if workflowStep.Title != nil && *workflowStep.Title != "" { stepInfoPtr.ID = *workflowStep.Title } stepInfoPtr.Version = stepIDData.Version if err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepDir := bitrise.BitriseWorkStepsDirPath stepYMLPth := path.Join(bitrise.BitriseWorkDirPath, "current_step.yml") // Activating the step if stepIDData.SteplibSource == "path" { log.Debugf("[BITRISE_CLI] - Local step found: (path:%s)", stepIDData.IDorURI) stepAbsLocalPth, err := pathutil.AbsPath(stepIDData.IDorURI) if err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } log.Debugln("stepAbsLocalPth:", stepAbsLocalPth, "|stepDir:", stepDir) if err := cmdex.CopyDir(stepAbsLocalPth, stepDir, true); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if err := cmdex.CopyFile(path.Join(stepAbsLocalPth, "step.yml"), stepYMLPth); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if len(stepIDData.IDorURI) > 20 { stepInfoPtr.Version = fmt.Sprintf("path:...%s", stringutil.MaxLastChars(stepIDData.IDorURI, 17)) } else { stepInfoPtr.Version = fmt.Sprintf("path:%s", stepIDData.IDorURI) } } else if stepIDData.SteplibSource == "git" { log.Debugf("[BITRISE_CLI] - Remote step, with direct git uri: (uri:%s) (tag-or-branch:%s)", stepIDData.IDorURI, stepIDData.Version) if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, stepDir, stepIDData.Version); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if err := cmdex.CopyFile(path.Join(stepDir, "step.yml"), stepYMLPth); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepInfoPtr.Version = fmt.Sprintf("git:...%s", stringutil.MaxLastChars(stepIDData.IDorURI, 10)) if stepIDData.Version != "" { stepInfoPtr.Version = stepInfoPtr.Version + "@" + stepIDData.Version } } else if stepIDData.SteplibSource == "_" { log.Debugf("[BITRISE_CLI] - Steplib independent step, with direct git uri: (uri:%s) (tag-or-branch:%s)", stepIDData.IDorURI, stepIDData.Version) // Steplib independent steps are completly defined in workflow stepYMLPth = "" if err := workflowStep.FillMissingDefaults(); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, stepDir, stepIDData.Version); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepInfoPtr.Version = "_" if stepIDData.Version != "" { if len(stepIDData.Version) > 20 { stepInfoPtr.Version = stepInfoPtr.Version + "@..." + stringutil.MaxLastChars(stepIDData.Version, 17) } else { stepInfoPtr.Version = stepInfoPtr.Version + "@" + stepIDData.Version } } } else if stepIDData.SteplibSource != "" { log.Debugf("[BITRISE_CLI] - Steplib (%s) step (id:%s) (version:%s) found, activating step", stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err := bitrise.StepmanSetup(stepIDData.SteplibSource); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepInfo, err := bitrise.StepmanStepInfo(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err != nil { if buildRunResults.IsStepLibUpdated(stepIDData.SteplibSource) { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } // May StepLib should be updated log.Info("Step info not found in StepLib (%s) -- Updating ...") if err := bitrise.StepmanUpdate(stepIDData.SteplibSource); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } buildRunResults.StepmanUpdates[stepIDData.SteplibSource]++ stepInfo, err = bitrise.StepmanStepInfo(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } } stepInfoPtr.Version = stepInfo.Version stepInfoPtr.Latest = stepInfo.Latest if err := bitrise.StepmanActivate(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version, stepDir, stepYMLPth); err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } else { log.Debugf("[BITRISE_CLI] - Step activated: (ID:%s) (version:%s)", stepIDData.IDorURI, stepIDData.Version) } } else { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, fmt.Errorf("Invalid stepIDData: No SteplibSource or LocalPath defined (%v)", stepIDData), isLastStep) continue } // Fill step info with default step info, if exist mergedStep := workflowStep if stepYMLPth != "" { specStep, err := bitrise.ReadSpecStep(stepYMLPth) log.Debugf("Spec read from YML: %#v\n", specStep) if err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } mergedStep, err = models.MergeStepWith(specStep, workflowStep) if err != nil { registerStepRunResults("", models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } } // Run step bitrise.PrintRunningStep(stepInfoPtr, idx) if mergedStep.RunIf != nil && *mergedStep.RunIf != "" { isRun, err := bitrise.EvaluateStepTemplateToBool(*mergedStep.RunIf, buildRunResults) if err != nil { registerStepRunResults(*mergedStep.RunIf, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if !isRun { registerStepRunResults(*mergedStep.RunIf, models.StepRunStatusCodeSkippedWithRunIf, 0, err, isLastStep) continue } } outEnvironments := []envmanModels.EnvironmentItemModel{} isAlwaysRun := stepmanModels.DefaultIsAlwaysRun if mergedStep.IsAlwaysRun != nil { isAlwaysRun = *mergedStep.IsAlwaysRun } else { log.Warn("Step (%s) mergedStep.IsAlwaysRun is nil, should not!", stepIDData.IDorURI) } if buildRunResults.IsBuildFailed() && !isAlwaysRun { registerStepRunResults(*mergedStep.RunIf, models.StepRunStatusCodeSkipped, 0, err, isLastStep) } else { exit, out, err := runStep(mergedStep, stepIDData, stepDir, *environments) outEnvironments = out if err != nil { if *mergedStep.IsSkippable { registerStepRunResults(*mergedStep.RunIf, models.StepRunStatusCodeFailedSkippable, exit, err, isLastStep) } else { registerStepRunResults(*mergedStep.RunIf, models.StepRunStatusCodeFailed, exit, err, isLastStep) } } else { registerStepRunResults(*mergedStep.RunIf, models.StepRunStatusCodeSuccess, 0, nil, isLastStep) *environments = append(*environments, outEnvironments...) } } } return buildRunResults }
func activateAndRunSteps(workflow models.WorkflowModel, defaultStepLibSource string, buildRunResults models.BuildRunResultsModel, environments *[]envmanModels.EnvironmentItemModel, isLastWorkflow bool) models.BuildRunResultsModel { log.Debugln("[BITRISE_CLI] - Activating and running steps") var stepStartTime time.Time registerStepRunResults := func(step stepmanModels.StepModel, resultCode, exitCode int, err error, isLastStep bool) { if step.Title == nil { log.Error("Step title is nil, should not happend!") step.Title = pointers.NewStringPtr("ERROR! Step title is nil!") } stepResults := models.StepRunResultsModel{ StepName: *step.Title, Status: resultCode, Idx: buildRunResults.ResultsCount(), RunTime: time.Now().Sub(stepStartTime), Error: err, ExitCode: exitCode, } switch resultCode { case models.StepRunStatusCodeSuccess: buildRunResults.SuccessSteps = append(buildRunResults.SuccessSteps, stepResults) break case models.StepRunStatusCodeFailed: log.Errorf("Step (%s) failed, error: (%v)", *step.Title, err) buildRunResults.FailedSteps = append(buildRunResults.FailedSteps, stepResults) break case models.StepRunStatusCodeFailedSkippable: log.Warnf("Step (%s) failed, but was marked as skippable, error: (%v)", *step.Title, err) buildRunResults.FailedSkippableSteps = append(buildRunResults.FailedSkippableSteps, stepResults) break case models.StepRunStatusCodeSkipped: log.Warnf("A previous step failed, and this step (%s) was not marked as IsAlwaysRun, skipped", *step.Title) buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break case models.StepRunStatusCodeSkippedWithRunIf: log.Warn("The step's (" + *step.Title + ") Run-If expression evaluated to false - skipping") log.Info("The Run-If expression was: ", colorstring.Blue(*step.RunIf)) buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break default: log.Error("Unkown result code") return } bitrise.PrintStepSummary(stepResults, isLastStep) } registerStepListItemRunResults := func(stepListItem models.StepListItemModel, resultCode, exitCode int, err error, isLastStep bool) { name := "" for key := range stepListItem { name = key break } stepResults := models.StepRunResultsModel{ StepName: name, Status: resultCode, Idx: buildRunResults.ResultsCount(), RunTime: time.Now().Sub(stepStartTime), Error: err, ExitCode: exitCode, } switch resultCode { case models.StepRunStatusCodeSuccess: buildRunResults.SuccessSteps = append(buildRunResults.SuccessSteps, stepResults) break case models.StepRunStatusCodeFailed: log.Errorf("Step (%s) failed, error: (%v)", name, err) buildRunResults.FailedSteps = append(buildRunResults.FailedSteps, stepResults) break case models.StepRunStatusCodeFailedSkippable: log.Warnf("Step (%s) failed, but was marked as skippable, error: (%v)", name, err) buildRunResults.FailedSkippableSteps = append(buildRunResults.FailedSkippableSteps, stepResults) break case models.StepRunStatusCodeSkipped: log.Warnf("A previous step failed, and this step (%s) was not marked as IsAlwaysRun, skipped", name) buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break case models.StepRunStatusCodeSkippedWithRunIf: log.Warn("The step's (" + name + ") Run-If expression evaluated to false - skipping") buildRunResults.SkippedSteps = append(buildRunResults.SkippedSteps, stepResults) break default: log.Error("Unkown result code") return } bitrise.PrintStepSummary(stepResults, isLastStep) } for idx, stepListItm := range workflow.Steps { stepStartTime = time.Now() isLastStep := isLastWorkflow && (idx == len(workflow.Steps)-1) if err := bitrise.SetBuildFailedEnv(buildRunResults.IsBuildFailed()); err != nil { log.Error("Failed to set Build Status envs") } compositeStepIDStr, workflowStep, err := models.GetStepIDStepDataPair(stepListItm) if err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepIDData, err := models.CreateStepIDDataFromString(compositeStepIDStr, defaultStepLibSource) if err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepVersionForInfoPrint := stepIDData.Version log.Debugf("[BITRISE_CLI] - Running Step: %#v", workflowStep) if err := bitrise.CleanupStepWorkDir(); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepDir := bitrise.BitriseWorkStepsDirPath stepYMLPth := path.Join(bitrise.BitriseWorkDirPath, "current_step.yml") if stepIDData.SteplibSource == "path" { log.Debugf("[BITRISE_CLI] - Local step found: (path:%s)", stepIDData.IDorURI) stepAbsLocalPth, err := pathutil.AbsPath(stepIDData.IDorURI) if err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } log.Debugln("stepAbsLocalPth:", stepAbsLocalPth, "|stepDir:", stepDir) if err := cmdex.CopyDir(stepAbsLocalPth, stepDir, true); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if err := cmdex.CopyFile(path.Join(stepAbsLocalPth, "step.yml"), stepYMLPth); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if len(stepIDData.IDorURI) > 20 { stepVersionForInfoPrint = fmt.Sprintf("path:...%s", stringutil.MaxLastChars(stepIDData.IDorURI, 17)) } else { stepVersionForInfoPrint = fmt.Sprintf("path:%s", stepIDData.IDorURI) } } else if stepIDData.SteplibSource == "git" { log.Debugf("[BITRISE_CLI] - Remote step, with direct git uri: (uri:%s) (tag-or-branch:%s)", stepIDData.IDorURI, stepIDData.Version) if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, stepDir, stepIDData.Version); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if err := cmdex.CopyFile(path.Join(stepDir, "step.yml"), stepYMLPth); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepVersionForInfoPrint = fmt.Sprintf("git:...%s", stringutil.MaxLastChars(stepIDData.IDorURI, 10)) if stepIDData.Version != "" { stepVersionForInfoPrint = stepVersionForInfoPrint + "@" + stepIDData.Version } } else if stepIDData.SteplibSource == "_" { log.Debugf("[BITRISE_CLI] - Steplib independent step, with direct git uri: (uri:%s) (tag-or-branch:%s)", stepIDData.IDorURI, stepIDData.Version) // Steplib independent steps are completly defined in workflow stepYMLPth = "" if err := workflowStep.FillMissingDefaults(); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if err := cmdex.GitCloneTagOrBranch(stepIDData.IDorURI, stepDir, stepIDData.Version); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepVersionForInfoPrint = "_" if stepIDData.Version != "" { if len(stepIDData.Version) > 20 { stepVersionForInfoPrint = stepVersionForInfoPrint + "@..." + stringutil.MaxLastChars(stepIDData.Version, 17) } else { stepVersionForInfoPrint = stepVersionForInfoPrint + "@" + stepIDData.Version } } } else if stepIDData.SteplibSource != "" { log.Debugf("[BITRISE_CLI] - Steplib (%s) step (id:%s) (version:%s) found, activating step", stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err := bitrise.StepmanSetup(stepIDData.SteplibSource); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } stepInfo, err := bitrise.StepmanStepInfo(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err != nil { if buildRunResults.IsStepLibUpdated(stepIDData.SteplibSource) { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } // May StepLib should be updated log.Info("Step info not found in StepLib (%s) -- Updating ...") if err := bitrise.StepmanUpdate(stepIDData.SteplibSource); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } buildRunResults.StepmanUpdates[stepIDData.SteplibSource]++ stepInfo, err = bitrise.StepmanStepInfo(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version) if err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } } stepVersionForInfoPrint = stepInfo.StepVersion if err := bitrise.StepmanActivate(stepIDData.SteplibSource, stepIDData.IDorURI, stepIDData.Version, stepDir, stepYMLPth); err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } else { log.Debugf("[BITRISE_CLI] - Step activated: (ID:%s) (version:%s)", stepIDData.IDorURI, stepIDData.Version) } } else { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, fmt.Errorf("Invalid stepIDData: No SteplibSource or LocalPath defined (%v)", stepIDData), isLastStep) continue } mergedStep := workflowStep if stepYMLPth != "" { specStep, err := bitrise.ReadSpecStep(stepYMLPth) log.Debugf("Spec read from YML: %#v\n", specStep) if err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } mergedStep, err = models.MergeStepWith(specStep, workflowStep) if err != nil { registerStepListItemRunResults(stepListItm, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } } // Run step bitrise.PrintRunningStep(*mergedStep.Title, stepVersionForInfoPrint, idx) if mergedStep.RunIf != nil && *mergedStep.RunIf != "" { isRun, err := bitrise.EvaluateStepTemplateToBool(*mergedStep.RunIf, buildRunResults) if err != nil { registerStepRunResults(mergedStep, models.StepRunStatusCodeFailed, 1, err, isLastStep) continue } if !isRun { registerStepRunResults(mergedStep, models.StepRunStatusCodeSkippedWithRunIf, 0, err, isLastStep) continue } } outEnvironments := []envmanModels.EnvironmentItemModel{} isAlwaysRun := stepmanModels.DefaultIsAlwaysRun if mergedStep.IsAlwaysRun != nil { isAlwaysRun = *mergedStep.IsAlwaysRun } else { log.Warn("Step (%s) mergedStep.IsAlwaysRun is nil, should not!", stepIDData.IDorURI) } if buildRunResults.IsBuildFailed() && !isAlwaysRun { registerStepRunResults(mergedStep, models.StepRunStatusCodeSkipped, 0, err, isLastStep) } else { exit, out, err := runStep(mergedStep, stepIDData, stepDir, *environments) outEnvironments = out if err != nil { if *mergedStep.IsSkippable { registerStepRunResults(mergedStep, models.StepRunStatusCodeFailedSkippable, exit, err, isLastStep) } else { registerStepRunResults(mergedStep, models.StepRunStatusCodeFailed, exit, err, isLastStep) } } else { registerStepRunResults(mergedStep, models.StepRunStatusCodeSuccess, 0, nil, isLastStep) *environments = append(*environments, outEnvironments...) } } } return buildRunResults }