// limitedLogAndRetry stops retrying after maxTimeout, failing the build.
func limitedLogAndRetry(buildupdater buildclient.BuildUpdater, maxTimeout time.Duration) controller.RetryFunc {
return func(obj interface{}, err error, retries controller.Retry) bool {
isFatal := strategy.IsFatal(err)
build := obj.(*buildapi.Build)
if !isFatal && time.Since(retries.StartTimestamp.Time) < maxTimeout {
glog.V(4).Infof("Retrying Build %s/%s with error: %v", build.Namespace, build.Name, err)
return true
}
build.Status.Phase = buildapi.BuildPhaseFailed
if !isFatal {
build.Status.Reason = buildapi.StatusReasonExceededRetryTimeout
build.Status.Message = buildapi.StatusMessageExceededRetryTimeout
}
build.Status.Message = errors.ErrorToSentence(err)
now := unversioned.Now()
build.Status.CompletionTimestamp = &now
glog.V(3).Infof("Giving up retrying Build %s/%s: %v", build.Namespace, build.Name, err)
utilruntime.HandleError(err)
if err := buildupdater.Update(build.Namespace, build); err != nil {
// retry update, but only on error other than NotFound
return !kerrors.IsNotFound(err)
}
return false
}
}
// Create constructs a BuildController
func (factory *BuildControllerFactory) Create() controller.RunnableController {
queue := cache.NewResyncableFIFO(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&buildLW{client: factory.OSClient}, &buildapi.Build{}, queue, 2*time.Minute).RunUntil(factory.Stop)
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(&kcoreclient.EventSinkImpl{Interface: factory.KubeClient.Core().Events("")})
client := ControllerClient{factory.KubeClient, factory.OSClient}
buildController := &buildcontroller.BuildController{
BuildUpdater: factory.BuildUpdater,
BuildLister: factory.BuildLister,
ImageStreamClient: client,
PodManager: client,
RunPolicies: policy.GetAllRunPolicies(factory.BuildLister, factory.BuildUpdater),
BuildStrategy: &typeBasedFactoryStrategy{
DockerBuildStrategy: factory.DockerBuildStrategy,
SourceBuildStrategy: factory.SourceBuildStrategy,
CustomBuildStrategy: factory.CustomBuildStrategy,
},
Recorder: eventBroadcaster.NewRecorder(kapi.EventSource{Component: "build-controller"}),
BuildDefaults: factory.BuildDefaults,
BuildOverrides: factory.BuildOverrides,
}
return &controller.RetryController{
Queue: queue,
RetryManager: controller.NewQueueRetryManager(
queue,
cache.MetaNamespaceKeyFunc,
limitedLogAndRetry(factory.BuildUpdater, 30*time.Minute),
flowcontrol.NewTokenBucketRateLimiter(1, 10)),
Handle: func(obj interface{}) error {
build := obj.(*buildapi.Build)
err := buildController.HandleBuild(build)
if err != nil {
// Update the build status message only if it changed.
if msg := errors.ErrorToSentence(err); build.Status.Message != msg {
// Set default Reason.
if len(build.Status.Reason) == 0 {
build.Status.Reason = buildapi.StatusReasonError
}
build.Status.Message = msg
if err := buildController.BuildUpdater.Update(build.Namespace, build); err != nil {
glog.V(2).Infof("Failed to update status message of Build %s/%s: %v", build.Namespace, build.Name, err)
}
buildController.Recorder.Eventf(build, kapi.EventTypeWarning, "HandleBuildError", "Build has error: %v", err)
}
}
return err
},
}
}