// limitedLogAndRetry stops retrying after maxTimeout, failing the build.
func limitedLogAndRetry(buildupdater buildclient.BuildUpdater, maxTimeout time.Duration) controller.RetryFunc {
return func(obj interface{}, err error, retries controller.Retry) bool {
isFatal := strategy.IsFatal(err)
build := obj.(*buildapi.Build)
if !isFatal && time.Since(retries.StartTimestamp.Time) < maxTimeout {
glog.V(4).Infof("Retrying Build %s/%s with error: %v", build.Namespace, build.Name, err)
return true
}
build.Status.Phase = buildapi.BuildPhaseFailed
if !isFatal {
build.Status.Reason = buildapi.StatusReasonExceededRetryTimeout
build.Status.Message = buildapi.StatusMessageExceededRetryTimeout
}
build.Status.Message = errors.ErrorToSentence(err)
now := unversioned.Now()
build.Status.CompletionTimestamp = &now
glog.V(3).Infof("Giving up retrying Build %s/%s: %v", build.Namespace, build.Name, err)
utilruntime.HandleError(err)
if err := buildupdater.Update(build.Namespace, build); err != nil {
// retry update, but only on error other than NotFound
return !kerrors.IsNotFound(err)
}
return false
}
}
// nextBuildPhase updates build with any appropriate changes, or returns an error if
// the change cannot occur. When returning nil, be sure to set build.Status and optionally
// build.Message.
func (bc *BuildController) nextBuildPhase(build *buildapi.Build) error {
// If a cancelling event was triggered for the build, update build status.
if build.Status.Cancelled {
glog.V(4).Infof("Cancelling build %s/%s.", build.Namespace, build.Name)
build.Status.Phase = buildapi.BuildPhaseCancelled
build.Status.Reason = ""
build.Status.Message = ""
return nil
}
// these builds are processed/updated/etc by the jenkins sync plugin
if build.Spec.Strategy.JenkinsPipelineStrategy != nil {
glog.V(4).Infof("Ignoring build with jenkins pipeline strategy")
return nil
}
// Set the output Docker image reference.
ref, err := bc.resolveOutputDockerImageReference(build)
if err != nil {
build.Status.Reason = buildapi.StatusReasonInvalidOutputReference
return err
}
build.Status.OutputDockerImageReference = ref
// Make a copy to avoid mutating the build from this point on.
copy, err := kapi.Scheme.Copy(build)
if err != nil {
return fmt.Errorf("unable to copy build: %v", err)
}
buildCopy := copy.(*buildapi.Build)
// TODO(rhcarvalho)
// The S2I and Docker builders expect build.Spec.Output.To to contain a
// resolved reference to a Docker image. Since build.Spec is immutable, we
// change a copy (that is never persisted) and pass it to
// bc.BuildStrategy.CreateBuildPod. We should make the builders use
// build.Status.OutputDockerImageReference, what will make copying the build
// unnecessary.
if build.Spec.Output.To != nil && len(build.Spec.Output.To.Name) != 0 {
buildCopy.Spec.Output.To = &kapi.ObjectReference{
Kind: "DockerImage",
Name: ref,
}
}
// Invoke the strategy to get a build pod.
podSpec, err := bc.BuildStrategy.CreateBuildPod(buildCopy)
if err != nil {
build.Status.Reason = buildapi.StatusReasonCannotCreateBuildPodSpec
if strategy.IsFatal(err) {
return strategy.FatalError(fmt.Sprintf("failed to create a build pod spec for build %s/%s: %v", build.Namespace, build.Name, err))
}
return fmt.Errorf("failed to create a build pod spec for build %s/%s: %v", build.Namespace, build.Name, err)
}
glog.V(4).Infof("Pod %s for build %s/%s is about to be created", podSpec.Name, build.Namespace, build.Name)
if _, err := bc.PodManager.CreatePod(build.Namespace, podSpec); err != nil {
if errors.IsAlreadyExists(err) {
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "failedCreate", "Pod already exists: %s/%s", podSpec.Namespace, podSpec.Name)
glog.V(4).Infof("Build pod already existed: %#v", podSpec)
return nil
}
// Log an event if the pod is not created (most likely due to quota denial).
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "FailedCreate", "Error creating: %v", err)
build.Status.Reason = buildapi.StatusReasonCannotCreateBuildPod
return fmt.Errorf("failed to create build pod: %v", err)
}
if build.Annotations == nil {
build.Annotations = make(map[string]string)
}
build.Annotations[buildapi.BuildPodNameAnnotation] = podSpec.Name
glog.V(4).Infof("Created pod for build: %#v", podSpec)
// Set the build phase, which will be persisted.
build.Status.Phase = buildapi.BuildPhasePending
build.Status.Reason = ""
build.Status.Message = ""
return nil
}
// nextBuildPhase updates build with any appropriate changes, or returns an error if
// the change cannot occur. When returning nil, be sure to set build.Status and optionally
// build.Message.
func (bc *BuildController) nextBuildPhase(build *buildapi.Build) error {
// If a cancelling event was triggered for the build, update build status.
if build.Status.Cancelled {
glog.V(4).Infof("Cancelling build %s/%s.", build.Namespace, build.Name)
build.Status.Phase = buildapi.BuildPhaseCancelled
return nil
}
// Set the output Docker image reference.
ref, err := bc.resolveOutputDockerImageReference(build)
if err != nil {
build.Status.Reason = buildapi.StatusReasonInvalidOutputReference
build.Status.Message = buildapi.StatusMessageInvalidOutputRef
return err
}
build.Status.OutputDockerImageReference = ref
// Make a copy to avoid mutating the build from this point on.
copy, err := kapi.Scheme.Copy(build)
if err != nil {
return fmt.Errorf("unable to copy build: %v", err)
}
buildCopy := copy.(*buildapi.Build)
// TODO(rhcarvalho)
// The S2I and Docker builders expect build.Spec.Output.To to contain a
// resolved reference to a Docker image. Since build.Spec is immutable, we
// change a copy (that is never persisted) and pass it to
// bc.BuildStrategy.CreateBuildPod. We should make the builders use
// build.Status.OutputDockerImageReference, what will make copying the build
// unnecessary.
if build.Spec.Output.To != nil && len(build.Spec.Output.To.Name) != 0 {
buildCopy.Spec.Output.To = &kapi.ObjectReference{
Kind: "DockerImage",
Name: ref,
}
}
// Invoke the strategy to get a build pod.
podSpec, err := bc.BuildStrategy.CreateBuildPod(buildCopy)
if err != nil {
build.Status.Reason = buildapi.StatusReasonCannotCreateBuildPodSpec
build.Status.Message = buildapi.StatusMessageCannotCreateBuildPodSpec
if strategy.IsFatal(err) {
return strategy.FatalError(fmt.Sprintf("failed to create a build pod spec for build %s/%s: %v", build.Namespace, build.Name, err))
}
return fmt.Errorf("failed to create a build pod spec for build %s/%s: %v", build.Namespace, build.Name, err)
}
if err := bc.BuildDefaults.ApplyDefaults(podSpec); err != nil {
return fmt.Errorf("failed to apply build defaults for build %s/%s: %v", build.Namespace, build.Name, err)
}
if err := bc.BuildOverrides.ApplyOverrides(podSpec); err != nil {
return fmt.Errorf("failed to apply build overrides for build %s/%s: %v", build.Namespace, build.Name, err)
}
glog.V(4).Infof("Pod %s for build %s/%s is about to be created", podSpec.Name, build.Namespace, build.Name)
if _, err := bc.PodManager.CreatePod(build.Namespace, podSpec); err != nil {
if errors.IsAlreadyExists(err) {
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "FailedCreate", "Pod already exists: %s/%s", podSpec.Namespace, podSpec.Name)
glog.V(4).Infof("Build pod already existed: %#v", podSpec)
// If the existing pod was created before this build, switch to Error state.
existingPod, err := bc.PodManager.GetPod(podSpec.Namespace, podSpec.Name)
if err == nil && existingPod.CreationTimestamp.Before(build.CreationTimestamp) {
build.Status.Phase = buildapi.BuildPhaseError
build.Status.Reason = buildapi.StatusReasonBuildPodExists
build.Status.Message = buildapi.StatusMessageBuildPodExists
}
return nil
}
// Log an event if the pod is not created (most likely due to quota denial).
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "FailedCreate", "Error creating: %v", err)
build.Status.Reason = buildapi.StatusReasonCannotCreateBuildPod
build.Status.Message = buildapi.StatusMessageCannotCreateBuildPod
return fmt.Errorf("failed to create build pod: %v", err)
}
setBuildPodNameAnnotation(build, podSpec.Name)
glog.V(4).Infof("Created pod for build: %#v", podSpec)
// Set the build phase, which will be persisted.
build.Status.Phase = buildapi.BuildPhasePending
build.Status.Reason = ""
build.Status.Message = ""
return nil
}