// Munge is the workhorse the will actually make updates to the PR
func (StaleGreenCI) Munge(obj *github.MungeObject) {
if !obj.IsPR() {
return
}
if !obj.HasLabel(lgtmLabel) {
return
}
if mergeable, err := obj.IsMergeable(); !mergeable || err != nil {
return
}
if !obj.IsStatusSuccess(requiredContexts) {
return
}
for _, context := range requiredContexts {
statusTime := obj.GetStatusTime(context)
if statusTime == nil {
glog.Errorf("%d: unable to determine time %q context was set", *obj.Issue.Number, context)
return
}
if time.Since(*statusTime) > staleGreenCIHours*time.Hour {
obj.WriteComment(greenMsgBody)
err := obj.WaitForPending(requiredContexts)
if err != nil {
glog.Errorf("Failed waiting for PR to start testing: %v", err)
}
return
}
}
}
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) {
err := obj.Refresh()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
if !sq.validForMerge(obj) {
return
}
if obj.HasLabel(e2eNotRequiredLabel) {
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged)
return
}
if err := obj.WriteComment(verifySafeToMergeBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
// Wait for the build to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
err = obj.WaitForPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to start testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to finish testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess([]string{sq.E2EStatusContext, sq.UnitStatusContext}); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return
}
if !sq.e2eStable() {
sq.SetMergeStatus(obj, e2eFailure)
return
}
obj.MergePR("submit-queue")
sq.updateMergeRate()
sq.SetMergeStatus(obj, merged)
return
}
func commentBeforeLastCI(obj *github.MungeObject, comment *githubapi.IssueComment) bool {
if !obj.IsStatusSuccess(requiredContexts) {
return false
}
if comment.CreatedAt == nil {
return false
}
commentTime := *comment.CreatedAt
for _, context := range requiredContexts {
statusTimeP := obj.GetStatusTime(context)
if statusTimeP == nil {
return false
}
statusTime := statusTimeP.Add(30 * time.Minute)
if commentTime.After(statusTime) {
return false
}
}
return true
}
// Munge is the workhorse the will actually make updates to the PR
func (StaleUnitTestMunger) Munge(obj *github.MungeObject) {
requiredContexts := []string{jenkinsUnitContext, jenkinsE2EContext}
if !obj.IsPR() {
return
}
if !obj.HasLabels([]string{"lgtm"}) {
return
}
if mergeable, err := obj.IsMergeable(); !mergeable || err != nil {
return
}
if !obj.IsStatusSuccess(requiredContexts) {
return
}
for _, context := range requiredContexts {
statusTime := obj.GetStatusTime(context)
if statusTime == nil {
glog.Errorf("%d: unable to determine time %q context was set", *obj.Issue.Number, context)
return
}
if time.Since(*statusTime) > staleHours*time.Hour {
msgFormat := `@k8s-bot test this
Tests are more than %d hours old. Re-running tests.`
msg := fmt.Sprintf(msgFormat, staleHours)
obj.WriteComment(msg)
err := obj.WaitForPending(requiredContexts)
if err != nil {
glog.Errorf("Failed waiting for PR to start testing: %v", err)
}
return
}
}
}
// Returns true if merge status changes, and false otherwise.
func (sq *SubmitQueue) retestPR(obj *github.MungeObject) bool {
if len(sq.RequiredRetestContexts) == 0 {
return false
}
if err := obj.WriteComment(retestBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return true
}
// Wait for the retest to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
atomic.AddInt32(&sq.prsTested, 1)
err := obj.WaitForPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to start testing: %v", err))
return true
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to finish testing: %v", err))
return true
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess(sq.RequiredRetestContexts); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return true
}
// no action taken.
return false
}
// validForMerge is the base logic about what PR can be automatically merged.
// PRs must pass this logic to be placed on the queue and they must pass this
// logic a second time to be retested/merged after they get to the top of
// the queue.
//
// If you update the logic PLEASE PLEASE PLEASE update serveMergeInfo() as well.
func (sq *SubmitQueue) validForMerge(obj *github.MungeObject) bool {
// Can't merge an issue!
if !obj.IsPR() {
return false
}
// Can't merge something already merged.
if m, err := obj.IsMerged(); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return false
} else if m {
sq.SetMergeStatus(obj, mergedByHand)
return false
}
if milestone := obj.Issue.Milestone; true {
title := ""
// Net set means the empty milestone, ""
if milestone != nil && milestone.Title != nil {
title = *milestone.Title
}
for _, blocked := range sq.DoNotMergeMilestones {
if title == blocked {
sq.SetMergeStatus(obj, unmergeableMilestone)
return false
}
}
}
// Must pass CLA checks
if !obj.HasLabel(claYesLabel) && !obj.HasLabel(claHumanLabel) && !obj.HasLabel(cncfClaYesLabel) {
sq.SetMergeStatus(obj, noCLA)
return false
}
// Obviously must be mergeable
if mergeable, err := obj.IsMergeable(); err != nil {
sq.SetMergeStatus(obj, undeterminedMergability)
return false
} else if !mergeable {
sq.SetMergeStatus(obj, unmergeable)
return false
}
// Validate the status information for this PR
if len(sq.RequiredStatusContexts) > 0 {
if ok := obj.IsStatusSuccess(sq.RequiredStatusContexts); !ok {
sq.SetMergeStatus(obj, ciFailure)
return false
}
}
if len(sq.RequiredRetestContexts) > 0 {
if ok := obj.IsStatusSuccess(sq.RequiredRetestContexts); !ok {
sq.SetMergeStatus(obj, ciFailure)
return false
}
}
// Clearly
if !(obj.HasLabel(lgtmLabel) || obj.HasLabel(approvedLabel)) {
sq.SetMergeStatus(obj, noLGTM)
return false
}
// PR cannot change since LGTM was added
lastModifiedTime := obj.LastModifiedTime()
// lgtmTime and approvedTime cannot both be nil at this point (see check above)
earliestApproved := getEarliestApprovedTime(obj)
if lastModifiedTime == nil || earliestApproved == nil {
glog.Errorf("PR %d was unable to determine when LGTM was added or when last modified", *obj.Issue.Number)
sq.SetMergeStatus(obj, unknown)
return false
}
if lastModifiedTime.After(*earliestApproved) {
sq.SetMergeStatus(obj, lgtmEarly)
return false
}
// PR cannot have the label which prevents merging.
if obj.HasLabel(doNotMergeLabel) {
sq.SetMergeStatus(obj, noMerge)
return false
}
return true
}
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) {
_, err := obj.RefreshPR()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown, true)
return
}
if m, err := obj.IsMerged(); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown, true)
return
} else if m {
sq.SetMergeStatus(obj, merged, true)
return
}
if mergeable, err := obj.IsMergeable(); err != nil {
sq.SetMergeStatus(obj, undeterminedMergability, true)
return
} else if !mergeable {
sq.SetMergeStatus(obj, unmergeable, true)
return
}
body := "@k8s-bot test this [submit-queue is verifying that this PR is safe to merge]"
if err := obj.WriteComment(body); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown, true)
return
}
// Wait for the build to start
sq.SetMergeStatus(obj, ghE2EWaitingStart, true)
err = obj.WaitForPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to start testing: %v", err)
sq.SetMergeStatus(obj, s, true)
return
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning, true)
err = obj.WaitForNotPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to finish testing: %v", err)
sq.SetMergeStatus(obj, s, true)
return
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess([]string{sq.E2EStatusContext, sq.UnitStatusContext}); !ok {
sq.SetMergeStatus(obj, ghE2EFailed, true)
return
}
if !sq.e2e.Stable() {
sq.flushGithubE2EQueue(e2eFailure)
sq.SetMergeStatus(obj, e2eFailure, true)
return
}
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged, true)
return
}
// Munge is the workhorse the will actually make updates to the PR
func (sq *SubmitQueue) Munge(obj *github.MungeObject) {
if !obj.IsPR() {
return
}
e2e := sq.e2e
userSet := sq.userWhitelist
if !obj.HasLabels([]string{claYes}) && !obj.HasLabels([]string{claHuman}) {
sq.SetMergeStatus(obj, noCLA, false)
return
}
if mergeable, err := obj.IsMergeable(); err != nil {
sq.SetMergeStatus(obj, undeterminedMergability, false)
return
} else if !mergeable {
sq.SetMergeStatus(obj, unmergeable, false)
return
}
// Validate the status information for this PR
contexts := sq.requiredStatusContexts(obj)
if ok := obj.IsStatusSuccess(contexts); !ok {
sq.SetMergeStatus(obj, ciFailure, false)
return
}
if !obj.HasLabel(sq.WhitelistOverride) && !userSet.Has(*obj.Issue.User.Login) {
if !obj.HasLabel(needsOKToMergeLabel) {
obj.AddLabels([]string{needsOKToMergeLabel})
body := "The author of this PR is not in the whitelist for merge, can one of the admins add the 'ok-to-merge' label?"
obj.WriteComment(body)
}
sq.SetMergeStatus(obj, needsok, false)
return
}
// Tidy up the issue list.
if obj.HasLabel(needsOKToMergeLabel) {
obj.RemoveLabel(needsOKToMergeLabel)
}
if !obj.HasLabels([]string{"lgtm"}) {
sq.SetMergeStatus(obj, noLGTM, false)
return
}
lastModifiedTime := obj.LastModifiedTime()
lgtmTime := obj.LabelTime("lgtm")
if lastModifiedTime == nil || lgtmTime == nil {
glog.Errorf("PR %d was unable to determine when LGTM was added or when last modified", *obj.Issue.Number)
sq.SetMergeStatus(obj, unknown, false)
return
}
if lastModifiedTime.After(*lgtmTime) {
sq.SetMergeStatus(obj, lgtmEarly, false)
return
}
if !e2e.Stable() {
sq.flushGithubE2EQueue(e2eFailure)
sq.SetMergeStatus(obj, e2eFailure, false)
return
}
// if there is a 'e2e-not-required' label, just merge it.
if obj.HasLabel(e2eNotRequiredLabel) {
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged, true)
return
}
added := false
sq.Lock()
if _, ok := sq.githubE2EQueue[*obj.Issue.Number]; !ok {
sq.githubE2EQueue[*obj.Issue.Number] = obj
sq.githubE2EWakeup <- true
added = true
}
sq.Unlock()
if added {
sq.SetMergeStatus(obj, ghE2EQueued, true)
}
return
}
// Returns true if we can discard the PR from the queue, false if we must keep it for later.
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) bool {
interruptedObj := sq.interruptedObj
sq.interruptedObj = nil
err := obj.Refresh()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return true
}
if !sq.validForMerge(obj) {
return true
}
if obj.HasLabel(retestNotRequiredLabel) {
sq.mergePullRequest(obj)
return true
}
if interruptedObj != nil {
if interruptedObj.hasSHAChanged() {
// This PR will have to be rested.
// Make sure we don't have higher priority first.
return false
}
glog.Infof("Skipping retest since head and base sha match previous attempt!")
atomic.AddInt32(&sq.retestsAvoided, 1)
} else {
if err := obj.WriteComment(retestBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return true
}
// Wait for the retest to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
err = obj.WaitForPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to start testing: %v", err))
return true
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending(sq.RequiredRetestContexts)
if err != nil {
sq.SetMergeStatus(obj, fmt.Sprintf("Failed waiting for PR to finish testing: %v", err))
return true
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess(sq.RequiredRetestContexts); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return true
}
}
if !sq.e2eStable(true) {
if sq.validForMerge(obj) {
sq.interruptedObj = newInterruptedObject(obj)
}
sq.SetMergeStatus(obj, e2eFailure)
return true
}
sq.mergePullRequest(obj)
return true
}
// Munge is the workhorse the will actually make updates to the PR
func (sq *SubmitQueue) Munge(obj *github.MungeObject) {
if !obj.IsPR() {
return
}
userSet := sq.userWhitelist
if !obj.HasLabels([]string{claYes}) && !obj.HasLabels([]string{claHuman}) {
sq.SetMergeStatus(obj, noCLA, false)
return
}
if mergeable, err := obj.IsMergeable(); err != nil {
sq.SetMergeStatus(obj, undeterminedMergability, false)
return
} else if !mergeable {
sq.SetMergeStatus(obj, unmergeable, false)
return
}
// Validate the status information for this PR
contexts := sq.requiredStatusContexts(obj)
if ok := obj.IsStatusSuccess(contexts); !ok {
sq.SetMergeStatus(obj, ciFailure, false)
return
}
if !obj.HasLabel(sq.WhitelistOverride) && !userSet.Has(*obj.Issue.User.Login) {
if !obj.HasLabel(needsOKToMergeLabel) {
obj.AddLabels([]string{needsOKToMergeLabel})
body := "The author of this PR is not in the whitelist for merge, can one of the admins add the 'ok-to-merge' label?"
obj.WriteComment(body)
}
sq.SetMergeStatus(obj, needsok, false)
return
}
// Tidy up the issue list.
if obj.HasLabel(needsOKToMergeLabel) {
obj.RemoveLabel(needsOKToMergeLabel)
}
if !obj.HasLabels([]string{"lgtm"}) {
sq.SetMergeStatus(obj, noLGTM, false)
return
}
lastModifiedTime := obj.LastModifiedTime()
lgtmTime := obj.LabelTime("lgtm")
if lastModifiedTime == nil || lgtmTime == nil {
glog.Errorf("PR %d was unable to determine when LGTM was added or when last modified", *obj.Issue.Number)
sq.SetMergeStatus(obj, unknown, false)
return
}
if lastModifiedTime.After(*lgtmTime) {
sq.SetMergeStatus(obj, lgtmEarly, false)
return
}
added := false
sq.Lock()
if _, ok := sq.githubE2EQueue[*obj.Issue.Number]; !ok {
added = true
}
// Add this most-recent object in place of the existing object. It will
// have more up2date information. Even though we explicitly refresh the
// PR information before do anything with it, this allow things like the
// queue order to change dynamically as labels are added/removed.
sq.githubE2EQueue[*obj.Issue.Number] = obj
sq.Unlock()
if added {
sq.SetMergeStatus(obj, ghE2EQueued, true)
}
return
}
// validForMerge is the base logic about what PR can be automatically merged.
// PRs must pass this logic to be placed on the queue and they must pass this
// logic a second time to be retested/merged after they get to the top of
// the queue.
//
// If you update the logic PLEASE PLEASE PLEASE update serveMergeInfo() as well.
func (sq *SubmitQueue) validForMerge(obj *github.MungeObject) bool {
// Can't merge an issue!
if !obj.IsPR() {
return false
}
// Can't merge something already merged.
if m, err := obj.IsMerged(); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return false
} else if m {
sq.SetMergeStatus(obj, merged)
return false
}
userSet := sq.userWhitelist
// Must pass CLA checks
if !obj.HasLabel(claYesLabel) && !obj.HasLabel(claHumanLabel) {
sq.SetMergeStatus(obj, noCLA)
return false
}
// Obviously must be mergeable
if mergeable, err := obj.IsMergeable(); err != nil {
sq.SetMergeStatus(obj, undeterminedMergability)
return false
} else if !mergeable {
sq.SetMergeStatus(obj, unmergeable)
return false
}
// Validate the status information for this PR
contexts := sq.requiredStatusContexts(obj)
if ok := obj.IsStatusSuccess(contexts); !ok {
sq.SetMergeStatus(obj, ciFailure)
return false
}
// The user either must be on the whitelist or have ok-to-merge
if !obj.HasLabel(okToMergeLabel) && !userSet.Has(*obj.Issue.User.Login) {
if !obj.HasLabel(needsOKToMergeLabel) {
obj.AddLabels([]string{needsOKToMergeLabel})
obj.WriteComment(notInWhitelistBody)
}
sq.SetMergeStatus(obj, needsok)
return false
}
// Tidy up the issue list.
if obj.HasLabel(needsOKToMergeLabel) {
obj.RemoveLabel(needsOKToMergeLabel)
}
// Clearly
if !obj.HasLabel(lgtmLabel) {
sq.SetMergeStatus(obj, noLGTM)
return false
}
// PR cannot change since LGTM was added
lastModifiedTime := obj.LastModifiedTime()
lgtmTime := obj.LabelTime(lgtmLabel)
if lastModifiedTime == nil || lgtmTime == nil {
glog.Errorf("PR %d was unable to determine when LGTM was added or when last modified", *obj.Issue.Number)
sq.SetMergeStatus(obj, unknown)
return false
}
if lastModifiedTime.After(*lgtmTime) {
sq.SetMergeStatus(obj, lgtmEarly)
return false
}
// PR cannot have the label which prevents merging.
if obj.HasLabel(doNotMergeLabel) {
sq.SetMergeStatus(obj, noMerge)
return false
}
return true
}
func (sq *SubmitQueue) doGithubE2EAndMerge(obj *github.MungeObject) {
err := obj.Refresh()
if err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
if !sq.validForMerge(obj) {
return
}
if obj.HasLabel(e2eNotRequiredLabel) {
obj.MergePR("submit-queue")
sq.SetMergeStatus(obj, merged)
return
}
// See if we can skip the retest.
headSHA, baseRef, gotHeadSHA := obj.GetHeadAndBase()
baseSHA := ""
gotBaseSHA := false
if gotHeadSHA {
baseSHA, gotBaseSHA = obj.GetSHAFromRef(baseRef)
}
maySkipTest := gotHeadSHA && gotBaseSHA &&
sq.interruptedMergeBaseSHA == baseSHA &&
sq.interruptedMergeHeadSHA == headSHA
if maySkipTest {
glog.Infof("Skipping retest since head and base sha match previous attempt!")
atomic.AddInt32(&sq.retestsAvoided, 1)
} else {
if err := obj.WriteComment(verifySafeToMergeBody); err != nil {
glog.Errorf("%d: unknown err: %v", *obj.Issue.Number, err)
sq.SetMergeStatus(obj, unknown)
return
}
// Wait for the build to start
sq.SetMergeStatus(obj, ghE2EWaitingStart)
err = obj.WaitForPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to start testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// re-get the base SHA in case something merged between us checking and
// starting the tests.
if gotHeadSHA {
baseSHA, gotBaseSHA = obj.GetSHAFromRef(baseRef)
}
// Wait for the status to go back to something other than pending
sq.SetMergeStatus(obj, ghE2ERunning)
err = obj.WaitForNotPending([]string{sq.E2EStatusContext, sq.UnitStatusContext})
if err != nil {
s := fmt.Sprintf("Failed waiting for PR to finish testing: %v", err)
sq.SetMergeStatus(obj, s)
return
}
// Check if the thing we care about is success
if ok := obj.IsStatusSuccess([]string{sq.E2EStatusContext, sq.UnitStatusContext}); !ok {
sq.SetMergeStatus(obj, ghE2EFailed)
return
}
}
if !sq.e2eStable(true) {
if gotHeadSHA && gotBaseSHA {
sq.interruptedMergeBaseSHA = baseSHA
sq.interruptedMergeHeadSHA = headSHA
}
sq.SetMergeStatus(obj, e2eFailure)
return
}
obj.MergePR("submit-queue")
sq.updateMergeRate()
sq.SetMergeStatus(obj, merged)
return
}