// The FetchRevisions method is used by a RepoTracker to run the pipeline for // tracking repositories. It performs everything from polling the repository to // persisting any changes retrieved from the repository reference. func (repoTracker *RepoTracker) FetchRevisions(numNewRepoRevisionsToFetch int) ( err error) { settings := repoTracker.Settings projectRef := repoTracker.ProjectRef projectIdentifier := projectRef.String() if !projectRef.Enabled { evergreen.Logger.Logf(slogger.INFO, "Skipping disabled project “%v”", projectRef) return nil } repository, err := model.FindRepository(projectIdentifier) if err != nil { return fmt.Errorf("error finding repository '%v': %v", projectIdentifier, err) } var revisions []model.Revision var lastRevision string if repository != nil { lastRevision = repository.LastRevision } if lastRevision == "" { // if this is the first time we're running the tracker for this project, // fetch the most recent `numNewRepoRevisionsToFetch` revisions evergreen.Logger.Logf(slogger.INFO, "No last recorded repository revision "+ "for “%v”. Proceeding to fetch most recent %v revisions", projectRef, numNewRepoRevisionsToFetch) revisions, err = repoTracker.GetRecentRevisions(numNewRepoRevisionsToFetch) } else { evergreen.Logger.Logf(slogger.INFO, "Last recorded repository revision for "+ "“%v” is “%v”", projectRef, lastRevision) revisions, err = repoTracker.GetRevisionsSince(lastRevision, settings.RepoTracker.MaxRepoRevisionsToSearch) } if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error fetching revisions for "+ "repository “%v”: %v", projectRef, err) repoTracker.sendFailureNotification(lastRevision, err) return nil } var lastVersion *version.Version if len(revisions) > 0 { lastVersion, err = repoTracker.StoreRevisions(revisions) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error storing revisions for "+ "repository %v: %v", projectRef, err) return err } err = model.UpdateLastRevision(lastVersion.Identifier, lastVersion.Revision) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error updating last revision for "+ "repository %v: %v", projectRef, err) return err } } else { lastVersion, err = version.FindOne(version.ByMostRecentForRequester(projectIdentifier, evergreen.RepotrackerVersionRequester)) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error getting most recent version for "+ "repository %v: %v", projectRef, err) return err } } if lastVersion == nil { evergreen.Logger.Logf(slogger.WARN, "no version to activate for repository %v", projectIdentifier) return nil } err = repoTracker.activateElapsedBuilds(lastVersion) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error activating variants: %v", err) return err } return nil }
// GetProjectConfig fetches the project configuration for a given repository // returning a remote config if the project references a remote repository // configuration file - via the Identifier. Otherwise it defaults to the local // project file func (repoTracker *RepoTracker) GetProjectConfig(revision string) ( project *model.Project, err error) { projectRef := repoTracker.ProjectRef if projectRef.LocalConfig != "" { // return the Local config from the project Ref. return model.FindProject("", projectRef) } project, err = repoTracker.GetRemoteConfig(revision) if err != nil { // Only create a stub version on API request errors that pertain // to actually fetching a config. Those errors currently include: // thirdparty.APIRequestError, thirdparty.FileNotFoundError and // thirdparty.YAMLFormatError _, apiReqErr := err.(thirdparty.APIRequestError) _, ymlFmtErr := err.(thirdparty.YAMLFormatError) _, noFileErr := err.(thirdparty.FileNotFoundError) if apiReqErr || noFileErr || ymlFmtErr { // If there's an error getting the remote config, e.g. because it // does not exist, we treat this the same as when the remote config // is invalid - but add a different error message message := fmt.Sprintf("error fetching project “%v” configuration "+ "data at revision “%v” (remote path=“%v”): %v", projectRef.Identifier, revision, projectRef.RemotePath, err) evergreen.Logger.Logf(slogger.ERROR, message) return nil, projectConfigError{[]string{message}} } // If we get here then we have an infrastructural error - e.g. // a thirdparty.APIUnmarshalError (indicating perhaps an API has // changed), a thirdparty.ResponseReadError(problem reading an // API response) or a thirdparty.APIResponseError (nil API // response) - or encountered a problem in fetching a local // configuration file. At any rate, this is bad enough that we // want to send a notification instead of just creating a stub // version. var lastRevision string repository, fErr := model.FindRepository(projectRef.Identifier) if fErr != nil || repository == nil { evergreen.Logger.Logf(slogger.ERROR, "error finding "+ "repository '%v': %v", projectRef.Identifier, fErr) } else { lastRevision = repository.LastRevision } repoTracker.sendFailureNotification(lastRevision, err) return nil, err } // check if project config is valid errs := validator.CheckProjectSyntax(project) if len(errs) != 0 { // We have syntax errors in the project. // Format them, as we need to store + display them to the user var message string var projectParseErrors []string for _, configError := range errs { message += fmt.Sprintf("\n\t=> %v", configError) projectParseErrors = append(projectParseErrors, configError.Error()) } evergreen.Logger.Logf(slogger.ERROR, "error validating project '%v' "+ "configuration at revision '%v': %v", projectRef.Identifier, revision, message) return nil, projectConfigError{projectParseErrors} } return project, nil }