// Munge is the workhorse the will actually make updates to the PR
func (StaleGreenCI) Munge(obj *github.MungeObject) {
if !obj.IsPR() {
return
}
if !obj.HasLabel(lgtmLabel) {
return
}
if mergeable, err := obj.IsMergeable(); !mergeable || err != nil {
return
}
if !obj.IsStatusSuccess(requiredContexts) {
return
}
for _, context := range requiredContexts {
statusTime := obj.GetStatusTime(context)
if statusTime == nil {
glog.Errorf("%d: unable to determine time %q context was set", *obj.Issue.Number, context)
return
}
if time.Since(*statusTime) > staleGreenCIHours*time.Hour {
obj.WriteComment(greenMsgBody)
err := obj.WaitForPending(requiredContexts)
if err != nil {
glog.Errorf("Failed waiting for PR to start testing: %v", err)
}
return
}
}
}
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) {
err := obj.Refresh()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
if !sq.validForMerge(obj) {
return
}
if obj.HasLabel(e2eNotRequiredLabel) {
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged)
return
}
if err := obj.WriteComment(verifySafeToMergeBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
// Wait for the build to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
err = obj.WaitForPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to start testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to finish testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess([]string{sq.E2EStatusContext, sq.UnitStatusContext}); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return
}
if !sq.e2eStable() {
sq.SetMergeStatus(obj, e2eFailure)
return
}
obj.MergePR("submit-queue")
sq.updateMergeRate()
sq.SetMergeStatus(obj, merged)
return
}
// Munge is the workhorse the will actually make updates to the PR
func (StaleUnitTestMunger) Munge(obj *github.MungeObject) {
requiredContexts := []string{jenkinsUnitContext, jenkinsE2EContext}
if !obj.IsPR() {
return
}
if !obj.HasLabels([]string{"lgtm"}) {
return
}
if mergeable, err := obj.IsMergeable(); !mergeable || err != nil {
return
}
if !obj.IsStatusSuccess(requiredContexts) {
return
}
for _, context := range requiredContexts {
statusTime := obj.GetStatusTime(context)
if statusTime == nil {
glog.Errorf("%d: unable to determine time %q context was set", *obj.Issue.Number, context)
return
}
if time.Since(*statusTime) > staleHours*time.Hour {
msgFormat := `@k8s-bot test this
Tests are more than %d hours old. Re-running tests.`
msg := fmt.Sprintf(msgFormat, staleHours)
obj.WriteComment(msg)
err := obj.WaitForPending(requiredContexts)
if err != nil {
glog.Errorf("Failed waiting for PR to start testing: %v", err)
}
return
}
}
}
// Returns true if merge status changes, and false otherwise.
func (sq *SubmitQueue) retestPR(obj *github.MungeObject) bool {
if len(sq.RequiredRetestContexts) == 0 {
return false
}
if err := obj.WriteComment(retestBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return true
}
// Wait for the retest to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
atomic.AddInt32(&sq.prsTested, 1)
err := obj.WaitForPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to start testing: %v", err))
return true
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to finish testing: %v", err))
return true
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess(sq.RequiredRetestContexts); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return true
}
// no action taken.
return false
}
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) {
_, err := obj.RefreshPR()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown, true)
return
}
if m, err := obj.IsMerged(); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown, true)
return
} else if m {
sq.SetMergeStatus(obj, merged, true)
return
}
if mergeable, err := obj.IsMergeable(); err != nil {
sq.SetMergeStatus(obj, undeterminedMergability, true)
return
} else if !mergeable {
sq.SetMergeStatus(obj, unmergeable, true)
return
}
body := "@k8s-bot test this [submit-queue is verifying that this PR is safe to merge]"
if err := obj.WriteComment(body); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown, true)
return
}
// Wait for the build to start
sq.SetMergeStatus(obj, ghE2EWaitingStart, true)
err = obj.WaitForPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to start testing: %v", err)
sq.SetMergeStatus(obj, s, true)
return
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning, true)
err = obj.WaitForNotPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to finish testing: %v", err)
sq.SetMergeStatus(obj, s, true)
return
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess([]string{sq.E2EStatusContext, sq.UnitStatusContext}); !ok {
sq.SetMergeStatus(obj, ghE2EFailed, true)
return
}
if !sq.e2e.Stable() {
sq.flushGithubE2EQueue(e2eFailure)
sq.SetMergeStatus(obj, e2eFailure, true)
return
}
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged, true)
return
}
// Returns true if we can discard the PR from the queue, false if we must keep it for later.
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) bool {
interruptedObj := sq.interruptedObj
sq.interruptedObj = nil
err := obj.Refresh()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return true
}
if !sq.validForMerge(obj) {
return true
}
if obj.HasLabel(retestNotRequiredLabel) {
sq.mergePullRequest(obj)
return true
}
if interruptedObj != nil {
if interruptedObj.hasSHAChanged() {
// This PR will have to be rested.
// Make sure we don't have higher priority first.
return false
}
glog.Infof("Skipping retest since head and base sha match previous attempt!")
atomic.AddInt32(&sq.retestsAvoided, 1)
} else {
if err := obj.WriteComment(retestBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return true
}
// Wait for the retest to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
err = obj.WaitForPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to start testing: %v", err))
return true
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to finish testing: %v", err))
return true
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess(sq.RequiredRetestContexts); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return true
}
}
if !sq.e2eStable(true) {
if sq.validForMerge(obj) {
sq.interruptedObj = newInterruptedObject(obj)
}
sq.SetMergeStatus(obj, e2eFailure)
return true
}
sq.mergePullRequest(obj)
return true
}
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) {
err := obj.Refresh()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
if !sq.validForMerge(obj) {
return
}
if obj.HasLabel(e2eNotRequiredLabel) {
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged)
return
}
// See if we can skip the retest.
headSHA, baseRef, gotHeadSHA := obj.GetHeadAndBase()
baseSHA := ""
gotBaseSHA := false
if gotHeadSHA {
baseSHA, gotBaseSHA = obj.GetSHAFromRef(baseRef)
}
maySkipTest := gotHeadSHA && gotBaseSHA &&
sq.interruptedMergeBaseSHA == baseSHA &&
sq.interruptedMergeHeadSHA == headSHA
if maySkipTest {
glog.Infof("Skipping retest since head and base sha match previous attempt!")
atomic.AddInt32(&sq.retestsAvoided, 1)
} else {
if err := obj.WriteComment(verifySafeToMergeBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
// Wait for the build to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
err = obj.WaitForPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to start testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// re-get the base SHA in case something merged between us checking and
// starting the tests.
if gotHeadSHA {
baseSHA, gotBaseSHA = obj.GetSHAFromRef(baseRef)
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to finish testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess([]string{sq.E2EStatusContext, sq.UnitStatusContext}); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return
}
}
if !sq.e2eStable(true) {
if gotHeadSHA && gotBaseSHA {
sq.interruptedMergeBaseSHA = baseSHA
sq.interruptedMergeHeadSHA = headSHA
}
sq.SetMergeStatus(obj, e2eFailure)
return
}
obj.MergePR("submit-queue")
sq.updateMergeRate()
sq.SetMergeStatus(obj, merged)
return
}