func TestValidateLGTMAfterPush(t *testing.T) {
tests := []struct {
issueEvents []github.IssueEvent
commits []github.RepositoryCommit
shouldPass bool
}{
{
issueEvents: NewLGTMEvents(), // Label >= time.Unix(10)
commits: Commits(), // Modified at time.Unix(7), 8, and 9
shouldPass: true,
},
{
issueEvents: OldLGTMEvents(), // Label <= time.Unix(8)
commits: Commits(), // Modified at time.Unix(7), 8, and 9
shouldPass: false,
},
{
issueEvents: OverlappingLGTMEvents(), // Labeled at 8, 9, and 10
commits: Commits(), // Modified at time.Unix(7), 8, and 9
shouldPass: true,
},
}
for testNum, test := range tests {
config := &github_util.Config{}
client, server, _ := github_test.InitServer(t, nil, nil, test.issueEvents, test.commits, nil)
config.Org = "o"
config.Project = "r"
config.SetClient(client)
obj := github_util.TestObject(config, BareIssue(), nil, nil, nil)
if _, err := obj.GetCommits(); err != nil {
t.Errorf("Unexpected error getting filled commits: %v", err)
}
if _, err := obj.GetEvents(); err != nil {
t.Errorf("Unexpected error getting events commits: %v", err)
}
lastModifiedTime := obj.LastModifiedTime()
lgtmTime := obj.LabelTime("lgtm")
if lastModifiedTime == nil || lgtmTime == nil {
t.Errorf("unexpected lastModifiedTime or lgtmTime == nil")
}
ok := !lastModifiedTime.After(*lgtmTime)
if ok != test.shouldPass {
t.Errorf("%d: expected: %v, saw: %v", testNum, test.shouldPass, ok)
}
server.Close()
}
}
func TestMunge(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
tests := []struct {
name string // because when the fail, counting is hard
pr *github.PullRequest
issue *github.Issue
commits []github.RepositoryCommit
events []github.IssueEvent
ciStatus *github.CombinedStatus
jenkinsJob jenkins.Job
e2ePass bool
unitPass bool
mergeAfterQueued bool
reason string
state string // what the github status context should be for the PR HEAD
}{
// Should pass because the entire thing was run and good
{
name: "Test1",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
jenkinsJob: SuccessJenkins(),
e2ePass: true,
unitPass: true,
reason: merged,
state: "success",
},
// Should list as 'merged' but the merge should happen before it gets e2e tested
// and we should bail early instead of waiting for a test that will never come.
{
name: "Test2",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(),
ciStatus: SuccessStatus(),
jenkinsJob: SuccessJenkins(),
// The test should never run, but if it does, make sure it fails
mergeAfterQueued: true,
reason: merged,
state: "success",
},
// Should merge even though github ci failed because of dont-require-e2e
{
name: "Test3",
pr: ValidPR(),
issue: DontRequireGithubE2EIssue(),
ciStatus: GithubE2EFailStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
reason: merged,
state: "success",
},
// Should merge even though user not in whitelist because has ok-to-merge
{
name: "Test4",
pr: ValidPR(),
issue: UserNotInWhitelistOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
e2ePass: true,
unitPass: true,
reason: merged,
state: "success",
},
// Fail because PR can't automatically merge
{
name: "Test5",
pr: UnMergeablePR(),
issue: NoOKToMergeIssue(),
reason: unmergeable,
state: "pending",
},
// Fail because we don't know if PR can automatically merge
{
name: "Test6",
pr: UndeterminedMergeablePR(),
issue: NoOKToMergeIssue(),
reason: undeterminedMergability,
state: "pending",
},
// Fail because the "cla: yes" label was not applied
{
name: "Test7",
pr: ValidPR(),
issue: NoCLAIssue(),
reason: noCLA,
state: "pending",
},
// Fail because github CI tests have failed (or at least are not success)
{
name: "Test8",
pr: NonWhitelistUserPR(),
issue: NoOKToMergeIssue(),
reason: ciFailure,
state: "pending",
},
// Fail because the user is not in the whitelist and we don't have "ok-to-merge"
{
name: "Test9",
pr: ValidPR(),
issue: UserNotInWhitelistNoOKToMergeIssue(),
ciStatus: SuccessStatus(),
reason: needsok,
state: "pending",
},
// Fail because missing LGTM label
{
name: "Test10",
pr: ValidPR(),
issue: NoLGTMIssue(),
ciStatus: SuccessStatus(),
reason: noLGTM,
state: "pending",
},
// Fail because we can't tell if LGTM was added before the last change
{
name: "Test11",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
reason: unknown,
state: "failure",
},
// Fail because LGTM was added before the last change
{
name: "Test12",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: OldLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
reason: lgtmEarly,
state: "pending",
},
// Fail because jenkins instances are failing (whole submit queue blocks)
{
name: "Test13",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: FailJenkins(),
reason: e2eFailure,
state: "success",
},
// Fail because the second run of github e2e tests failed
{
name: "Test14",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(),
jenkinsJob: SuccessJenkins(),
reason: ghE2EFailed,
state: "pending",
},
// When we check the reason it may be queued or it may already have failed.
{
name: "Test15",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
reason: ghE2EQueued,
// The state is unpredictable. When it goes on the queue it is success.
// When it fails the build it is pending. So state depends on how far along
// this were when we checked. Thus just don't check it...
state: "",
},
// Fail because the second run of github e2e tests failed
{
name: "Test16",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
reason: ghE2EFailed,
state: "pending",
},
{
name: "Fail because E2E pass, but unit test fail",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
jenkinsJob: SuccessJenkins(),
e2ePass: true,
unitPass: false,
reason: ghE2EFailed,
state: "pending",
},
{
name: "Fail because E2E fail, but unit test pass",
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
jenkinsJob: SuccessJenkins(),
e2ePass: false,
unitPass: true,
reason: ghE2EFailed,
state: "pending",
},
}
for testNum, test := range tests {
issueNum := testNum + 1
issueNumStr := strconv.Itoa(issueNum)
test.issue.Number = &issueNum
client, server, mux := github_test.InitServer(t, test.issue, test.pr, test.events, test.commits, test.ciStatus)
config := &github_util.Config{}
config.Org = "o"
config.Project = "r"
config.SetClient(client)
// Don't wait so long for it to go pending or back
d := 250 * time.Millisecond
config.PendingWaitTime = &d
stateSet := ""
numJenkinsCalls := 0
// Respond with success to jenkins requests.
mux.HandleFunc("/job/foo/lastCompletedBuild/api/json", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(test.jenkinsJob)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
// There is no good spot for this, but this gets called
// before we queue the PR. So mark the PR as "merged".
// When the sq initializes, it will check the Jenkins status,
// so we don't want to modify the PR there. Instead we need
// to wait until the second time we check Jenkins, which happens
// we did the IsMerged() check.
numJenkinsCalls = numJenkinsCalls + 1
if numJenkinsCalls == 2 && test.mergeAfterQueued {
test.pr.Merged = boolPtr(true)
test.pr.Mergeable = nil
}
})
path := fmt.Sprintf("/repos/o/r/issues/%d/comments", issueNum)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Errorf("Unexpected method: %s", r.Method)
}
type comment struct {
Body string `json:"body"`
}
c := new(comment)
json.NewDecoder(r.Body).Decode(c)
msg := c.Body
if strings.HasPrefix(msg, "@k8s-bot test this") {
go fakeRunGithubE2ESuccess(test.ciStatus, test.e2ePass, test.unitPass)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(github.IssueComment{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
})
path = fmt.Sprintf("/repos/o/r/pulls/%d/merge", issueNum)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "PUT" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(github.PullRequestMergeResult{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
test.pr.Merged = boolPtr(true)
})
path = fmt.Sprintf("/repos/o/r/statuses/%s", *test.pr.Head.SHA)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Errorf("Unexpected method: %s", r.Method)
}
decoder := json.NewDecoder(r.Body)
var status github.RepoStatus
err := decoder.Decode(&status)
if err != nil {
t.Errorf("Unable to decode status: %v", err)
}
stateSet = *status.State
data, err := json.Marshal(status)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.WriteHeader(http.StatusOK)
w.Write(data)
test.pr.Merged = boolPtr(true)
})
sq := SubmitQueue{}
sq.RequiredStatusContexts = []string{jenkinsUnitContext}
sq.E2EStatusContext = jenkinsE2EContext
sq.UnitStatusContext = jenkinsUnitContext
sq.JenkinsHost = server.URL
sq.JenkinsJobs = []string{"foo"}
sq.WhitelistOverride = "ok-to-merge"
sq.Initialize(config)
sq.EachLoop()
sq.userWhitelist.Insert(whitelistUser)
obj := github_util.TestObject(config, test.issue, test.pr, test.commits, test.events)
sq.Munge(obj)
done := make(chan bool, 1)
go func(done chan bool) {
for {
if sq.prStatus[issueNumStr].Reason == test.reason {
done <- true
return
}
found := false
for _, status := range sq.statusHistory {
if status.Number == issueNum && status.Reason == test.reason {
found = true
break
}
}
if found {
done <- true
return
}
time.Sleep(1 * time.Millisecond)
}
}(done)
select {
case <-done:
case <-time.After(10 * time.Second):
t.Errorf("%d:%s timed out waiting expected reason=%q but got prStatus:%q history:%v", testNum, test.name, test.reason, sq.prStatus[issueNumStr].Reason, sq.statusHistory)
}
close(done)
server.Close()
if test.state != "" && test.state != stateSet {
t.Errorf("%d:%s state set to %q but expected %q", testNum, test.name, stateSet, test.state)
}
}
}
func TestSubmitQueue(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
tests := []struct {
name string // because when the fail, counting is hard
pr *github.PullRequest
issue *github.Issue
commits []*github.RepositoryCommit
events []*github.IssueEvent
ciStatus *github.CombinedStatus
lastBuildNumber int
gcsResult utils.FinishedFile
weakResults map[int]utils.FinishedFile
gcsJunit map[string][]byte
retest1Pass bool
retest2Pass bool
mergeAfterQueued bool
reason string
state string // what the github status context should be for the PR HEAD
emergencyMergeStop bool
isMerged bool
imHeadSHA string
imBaseSHA string
masterCommit *github.RepositoryCommit
retestsAvoided int // desired output
}{
// Should pass because the entire thing was run and good
{
name: "Test1",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
reason: merged,
state: "success",
isMerged: true,
},
// Entire thing was run and good, but emergency merge stop in progress
{
name: "Test1+emergencyStop",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
emergencyMergeStop: true,
isMerged: false,
reason: e2eFailure,
state: "success",
},
// Should pass without running tests because we had a previous run.
// TODO: Add a proper test to make sure we don't shuffle queue when we can just merge a PR
{
name: "Test1+prevsuccess",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
reason: merged,
state: "success",
isMerged: true,
retestsAvoided: 1,
imHeadSHA: "mysha", // Set by ValidPR
imBaseSHA: "mastersha",
masterCommit: MasterCommit(),
},
// Should list as 'merged' but the merge should happen before it gets e2e tested
// and we should bail early instead of waiting for a test that will never come.
{
name: "Test2",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(),
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
// The test should never run, but if it does, make sure it fails
mergeAfterQueued: true,
reason: mergedByHand,
state: "success",
},
// Should merge even though retest1Pass would have failed before of `retestNotRequiredLabel`
{
name: "merge because of retestNotRequired",
pr: ValidPR(),
issue: NoRetestIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: false,
retest2Pass: false,
reason: merged,
state: "success",
isMerged: true,
},
// Fail because PR can't automatically merge
{
name: "Test5",
pr: UnMergeablePR(),
issue: LGTMIssue(),
reason: unmergeable,
state: "pending",
// To avoid false errors in logs
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
},
// Fail because we don't know if PR can automatically merge
{
name: "Test6",
pr: UndeterminedMergeablePR(),
issue: LGTMIssue(),
reason: undeterminedMergability,
state: "pending",
// To avoid false errors in logs
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
},
// Fail because the claYesLabel label was not applied
{
name: "Test7",
pr: ValidPR(),
issue: NoCLAIssue(),
reason: noCLA,
state: "pending",
// To avoid false errors in logs
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
},
// Fail because github CI tests have failed (or at least are not success)
{
name: "Test8",
pr: ValidPR(),
issue: LGTMIssue(),
reason: ciFailure,
state: "pending",
// To avoid false errors in logs
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
},
// Fail because missing LGTM label
{
name: "Test10",
pr: ValidPR(),
issue: NoLGTMIssue(),
ciStatus: SuccessStatus(),
reason: noLGTM,
state: "pending",
// To avoid false errors in logs
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
},
// Fail because we can't tell if LGTM was added before the last change
{
name: "Test11",
pr: ValidPR(),
issue: LGTMIssue(),
ciStatus: SuccessStatus(),
reason: unknown,
state: "failure",
// To avoid false errors in logs
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
},
// Fail because LGTM was added before the last change
{
name: "Test12",
pr: ValidPR(),
issue: LGTMIssue(),
ciStatus: SuccessStatus(),
events: OldLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
reason: lgtmEarly,
state: "pending",
},
// Fail because jenkins instances are failing (whole submit queue blocks)
{
name: "Test13",
pr: ValidPR(),
issue: LGTMIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
lastBuildNumber: LastBuildNumber(),
gcsResult: FailGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
reason: ghE2EQueued,
state: "success",
},
// Fail because the second run of github e2e tests failed
{
name: "Test14",
pr: ValidPR(),
issue: LGTMIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
reason: ghE2EFailed,
state: "pending",
},
// When we check the reason it may be queued or it may already have failed.
{
name: "Test15",
pr: ValidPR(),
issue: LGTMIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
reason: ghE2EQueued,
// The state is unpredictable. When it goes on the queue it is success.
// When it fails the build it is pending. So state depends on how far along
// this were when we checked. Thus just don't check it...
state: "",
},
// Fail because the second run of github e2e tests failed
{
name: "Test16",
pr: ValidPR(),
issue: LGTMIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
reason: ghE2EFailed,
state: "pending",
},
{
name: "Fail because E2E pass, but unit test fail",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: false,
reason: ghE2EFailed,
state: "pending",
},
{
name: "Fail because E2E fail, but unit test pass",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: false,
retest2Pass: true,
reason: ghE2EFailed,
state: "pending",
},
{
name: "Fail because doNotMerge label is present",
pr: ValidPR(),
issue: DoNotMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
reason: noMerge,
state: "pending",
},
// Should fail because the 'do-not-merge-milestone' is set.
{
name: "Do Not Merge Milestone Set",
pr: ValidPR(),
issue: DoNotMergeMilestoneIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
reason: unmergeableMilestone,
state: "pending",
},
{
name: "Fail because retest status fail",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: RetestFailStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
reason: ciFailure,
state: "pending",
},
{
name: "Fail because noretest status fail",
pr: ValidPR(),
issue: LGTMIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: NoRetestFailStatus(),
lastBuildNumber: LastBuildNumber(),
gcsResult: SuccessGCS(),
weakResults: map[int]utils.FinishedFile{LastBuildNumber(): SuccessGCS()},
retest1Pass: true,
retest2Pass: true,
reason: ciFailure,
state: "pending",
},
// // Should pass even though last 'weakStable' build failed, as it wasn't "strong" failure
// // and because previous two builds succeeded.
// {
// name: "Test20",
// pr: ValidPR(),
// issue: LGTMIssue(),
// events: NewLGTMEvents(),
// commits: Commits(), // Modified at time.Unix(7), 8, and 9
// ciStatus: SuccessStatus(),
// lastBuildNumber: LastBuildNumber(),
// gcsResult: SuccessGCS(),
// weakResults: map[int]utils.FinishedFile{
// LastBuildNumber(): FailGCS(),
// LastBuildNumber() - 1: SuccessGCS(),
// LastBuildNumber() - 2: SuccessGCS(),
// },
// gcsJunit: map[string][]byte{
// "junit_01.xml": getJUnit(5, 0),
// "junit_02.xml": getJUnit(6, 0),
// "junit_03.xml": getJUnit(7, 0),
// },
// retest1Pass: true,
// retest2Pass: true,
// reason: merged,
// state: "success",
// },
// // Should fail because the failure of the weakStable job is a strong failure.
// {
// name: "Test21",
// pr: ValidPR(),
// issue: LGTMIssue(),
// events: NewLGTMEvents(),
// commits: Commits(), // Modified at time.Unix(7), 8, and 9
// ciStatus: SuccessStatus(),
// lastBuildNumber: LastBuildNumber(),
// gcsResult: SuccessGCS(),
// weakResults: map[int]utils.FinishedFile{
// LastBuildNumber(): FailGCS(),
// LastBuildNumber() - 1: SuccessGCS(),
// LastBuildNumber() - 2: SuccessGCS(),
// },
// gcsJunit: map[string][]byte{
// "junit_01.xml": getJUnit(5, 0),
// "junit_02.xml": getJUnit(6, 1),
// "junit_03.xml": getJUnit(7, 0),
// },
// retest1Pass: true,
// retest2Pass: true,
// reason: e2eFailure,
// state: "success",
// },
// // Should fail even though weakStable job weakly failed, because there was another failure in
// // previous two runs.
// {
// name: "Test22",
// pr: ValidPR(),
// issue: LGTMIssue(),
// events: NewLGTMEvents(),
// commits: Commits(), // Modified at time.Unix(7), 8, and 9
// ciStatus: SuccessStatus(),
// lastBuildNumber: LastBuildNumber(),
// gcsResult: SuccessGCS(),
// weakResults: map[int]utils.FinishedFile{
// LastBuildNumber(): FailGCS(),
// LastBuildNumber() - 1: SuccessGCS(),
// LastBuildNumber() - 2: FailGCS(),
// },
// gcsJunit: map[string][]byte{
// "junit_01.xml": getJUnit(5, 0),
// "junit_02.xml": getJUnit(6, 0),
// "junit_03.xml": getJUnit(7, 0),
// },
// retest1Pass: true,
// retest2Pass: true,
// reason: e2eFailure,
// state: "success",
// },
}
for testNum := range tests {
test := &tests[testNum]
fmt.Printf("---------Starting test %v (%v)---------------------\n", testNum, test.name)
issueNum := testNum + 1
issueNumStr := strconv.Itoa(issueNum)
test.issue.Number = &issueNum
client, server, mux := github_test.InitServer(t, test.issue, test.pr, test.events, test.commits, test.ciStatus, test.masterCommit, nil)
config := &github_util.Config{}
config.Org = "o"
config.Project = "r"
config.SetClient(client)
// Don't wait so long for it to go pending or back
d := 250 * time.Millisecond
config.PendingWaitTime = &d
stateSet := ""
wasMerged := false
numTestChecks := 0
path := "/bucket/logs/foo/latest-build.txt"
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
w.Write([]byte(strconv.Itoa(test.lastBuildNumber)))
// There is no good spot for this, but this gets called
// before we queue the PR. So mark the PR as "merged".
// When the sq initializes, it will check the Jenkins status,
// so we don't want to modify the PR there. Instead we need
// to wait until the second time we check Jenkins, which happens
// we did the IsMerged() check.
numTestChecks = numTestChecks + 1
if numTestChecks == 2 && test.mergeAfterQueued {
test.pr.Merged = boolPtr(true)
test.pr.Mergeable = nil
}
})
path = fmt.Sprintf("/bucket/logs/foo/%v/finished.json", test.lastBuildNumber)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(test.gcsResult)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
})
path = "/bucket/logs/bar/latest-build.txt"
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
w.Write([]byte(strconv.Itoa(test.lastBuildNumber)))
})
for buildNumber := range test.weakResults {
path = fmt.Sprintf("/bucket/logs/bar/%v/finished.json", buildNumber)
// workaround go for loop semantics
buildNumberCopy := buildNumber
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(test.weakResults[buildNumberCopy])
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
})
}
for junitFile, xml := range test.gcsJunit {
path = fmt.Sprintf("/bucket/logs/bar/%v/artifacts/%v", test.lastBuildNumber, junitFile)
// workaround go for loop semantics
xmlCopy := xml
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
w.Write(xmlCopy)
})
}
path = fmt.Sprintf("/repos/o/r/issues/%d/comments", issueNum)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
c := new(github.IssueComment)
json.NewDecoder(r.Body).Decode(c)
msg := *c.Body
if strings.HasPrefix(msg, "@"+jenkinsBotName+" test this") {
go fakeRunGithubE2ESuccess(test.ciStatus, test.retest1Pass, test.retest2Pass)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(github.IssueComment{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
return
}
if r.Method == "GET" {
w.WriteHeader(http.StatusOK)
data, err := json.Marshal([]github.IssueComment{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
return
}
t.Errorf("Unexpected method: %s", r.Method)
})
path = fmt.Sprintf("/repos/o/r/pulls/%d/merge", issueNum)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "PUT" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(github.PullRequestMergeResult{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
test.pr.Merged = boolPtr(true)
wasMerged = true
})
path = fmt.Sprintf("/repos/o/r/statuses/%s", *test.pr.Head.SHA)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Errorf("Unexpected method: %s", r.Method)
}
decoder := json.NewDecoder(r.Body)
var status github.RepoStatus
err := decoder.Decode(&status)
if err != nil {
t.Errorf("Unable to decode status: %v", err)
}
stateSet = *status.State
data, err := json.Marshal(status)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.WriteHeader(http.StatusOK)
w.Write(data)
})
sq := getTestSQ(true, config, server)
sq.setEmergencyMergeStop(test.emergencyMergeStop)
obj := github_util.TestObject(config, test.issue, test.pr, test.commits, test.events)
if test.imBaseSHA != "" && test.imHeadSHA != "" {
sq.interruptedObj = &submitQueueInterruptedObject{obj, test.imHeadSHA, test.imBaseSHA}
}
sq.Munge(obj)
done := make(chan bool, 1)
go func(done chan bool) {
for {
defer func() {
if r := recover(); r != nil {
t.Errorf("%d:%q panic'd likely writing to 'done' channel", testNum, test.name)
}
}()
reason := func() string {
sq.Mutex.Lock()
defer sq.Mutex.Unlock()
return sq.prStatus[issueNumStr].Reason
}
if reason() == test.reason {
done <- true
return
}
found := false
for _, status := range sq.statusHistory {
if status.Reason == test.reason {
found = true
break
}
}
if found {
done <- true
return
}
time.Sleep(1 * time.Millisecond)
}
}(done)
select {
case <-done:
case <-time.After(10 * time.Second):
t.Errorf("%d:%q timed out waiting expected reason=%q but got prStatus:%q history:%v", testNum, test.name, test.reason, sq.prStatus[issueNumStr].Reason, sq.statusHistory)
}
close(done)
server.Close()
if test.state != "" && test.state != stateSet {
t.Errorf("%d:%q state set to %q but expected %q", testNum, test.name, stateSet, test.state)
}
if test.isMerged != wasMerged {
t.Errorf("%d:%q PR merged = %v but wanted %v", testNum, test.name, wasMerged, test.isMerged)
}
if e, a := test.retestsAvoided, int(sq.retestsAvoided); e != a {
t.Errorf("%d:%q expected %v tests avoided but got %v", testNum, test.name, e, a)
}
}
}
func TestMunge(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
tests := []struct {
pr *github.PullRequest
issue *github.Issue
commits []github.RepositoryCommit
events []github.IssueEvent
ciStatus *github.CombinedStatus
jenkinsJob jenkins.Job
shouldPass bool
mergeAfterQueued bool
reasons []string
}{
// Should pass because the entire thing was run and good
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
ciStatus: SuccessStatus(),
jenkinsJob: SuccessJenkins(),
shouldPass: true,
reasons: []string{merged},
},
// Should list as 'merged' but the merge should happen before it gets e2e tested
// and we should bail early instead of waiting for a test that will never come.
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(),
ciStatus: SuccessStatus(),
jenkinsJob: SuccessJenkins(),
// The test should never run, but if it does, make sure it fails
shouldPass: false,
mergeAfterQueued: true,
reasons: []string{merged},
},
// Should merge even though github ci failed because of dont-require-e2e
{
pr: ValidPR(),
issue: DontRequireGithubE2EIssue(),
ciStatus: GithubE2EFailStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
reasons: []string{merged},
},
// Should merge even though user not in whitelist because has ok-to-merge
{
pr: ValidPR(),
issue: UserNotInWhitelistOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
shouldPass: true,
reasons: []string{merged},
},
// Fail because PR can't automatically merge
{
pr: UnMergeablePR(),
issue: NoOKToMergeIssue(),
reasons: []string{unmergeable},
},
// Fail because we don't know if PR can automatically merge
{
pr: UndeterminedMergeablePR(),
issue: NoOKToMergeIssue(),
reasons: []string{undeterminedMergability},
},
// Fail because the "cla: yes" label was not applied
{
pr: ValidPR(),
issue: NoCLAIssue(),
reasons: []string{noCLA},
},
// Fail because github CI tests have failed (or at least are not success)
{
pr: NonWhitelistUserPR(),
issue: NoOKToMergeIssue(),
reasons: []string{ciFailure},
},
// Fail because the user is not in the whitelist and we don't have "ok-to-merge"
{
pr: ValidPR(),
issue: UserNotInWhitelistNoOKToMergeIssue(),
ciStatus: SuccessStatus(),
reasons: []string{needsok},
},
// Fail because missing LGTM label
{
pr: ValidPR(),
issue: NoLGTMIssue(),
ciStatus: SuccessStatus(),
reasons: []string{noLGTM},
},
// Fail because we can't tell if LGTM was added before the last change
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
reasons: []string{unknown},
},
// Fail because LGTM was added before the last change
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: OldLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
reasons: []string{lgtmEarly},
},
// Fail because jenkins instances are failing (whole submit queue blocks)
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: FailJenkins(),
reasons: []string{e2eFailure},
},
// Fail because the second run of github e2e tests failed
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(),
jenkinsJob: SuccessJenkins(),
reasons: []string{ghE2EFailed},
},
// Should pass because the jenkins ci is green even tho shippable is pending.
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(),
ciStatus: JenkinsCIGreenShippablePendingStatus(),
jenkinsJob: SuccessJenkins(),
shouldPass: true,
reasons: []string{merged},
},
// Should pass because the shippable is green (no jenkins ci).
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
events: NewLGTMEvents(),
commits: Commits(),
ciStatus: ShippableGreenStatus(),
jenkinsJob: SuccessJenkins(),
shouldPass: true,
reasons: []string{merged},
},
// When we check the reason it may be queued or it may already have failed.
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
shouldPass: false,
reasons: []string{ghE2EQueued, ghE2EFailed},
},
// Fail because the second run of github e2e tests failed
{
pr: ValidPR(),
issue: NoOKToMergeIssue(),
ciStatus: SuccessStatus(),
events: NewLGTMEvents(),
commits: Commits(), // Modified at time.Unix(7), 8, and 9
jenkinsJob: SuccessJenkins(),
shouldPass: false,
reasons: []string{ghE2EFailed},
},
}
for testNum, test := range tests {
client, server, mux := github_test.InitServer(t, test.issue, test.pr, test.events, test.commits, test.ciStatus)
config := &github_util.Config{}
config.Org = "o"
config.Project = "r"
config.SetClient(client)
// Don't wait so long for it to go pending or back
d := 250 * time.Millisecond
config.PendingWaitTime = &d
numJenkinsCalls := 0
// Respond with success to jenkins requests.
mux.HandleFunc("/job/foo/lastCompletedBuild/api/json", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(test.jenkinsJob)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
// There is no good spot for this, but this gets called
// before we queue the PR. So mark the PR as "merged".
// When the sq initializes, it will check the Jenkins status,
// so we don't want to modify the PR there. Instead we need
// to wait until the second time we check Jenkins, which happens
// we did the IsMerged() check.
numJenkinsCalls = numJenkinsCalls + 1
if numJenkinsCalls == 2 && test.mergeAfterQueued {
test.pr.Merged = boolPtr(true)
test.pr.Mergeable = nil
}
})
mux.HandleFunc("/repos/o/r/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Errorf("Unexpected method: %s", r.Method)
}
type comment struct {
Body string `json:"body"`
}
c := new(comment)
json.NewDecoder(r.Body).Decode(c)
msg := c.Body
if strings.HasPrefix(msg, "@k8s-bot test this") {
go fakeRunGithubE2ESuccess(test.ciStatus, test.shouldPass)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(github.IssueComment{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
})
mux.HandleFunc("/repos/o/r/pulls/1/merge", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "PUT" {
t.Errorf("Unexpected method: %s", r.Method)
}
w.WriteHeader(http.StatusOK)
data, err := json.Marshal(github.PullRequestMergeResult{})
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
w.Write(data)
test.pr.Merged = boolPtr(true)
})
sq := SubmitQueue{}
sq.RequiredStatusContexts = []string{claContext}
sq.DontRequireE2ELabel = "e2e-not-required"
sq.E2EStatusContext = gceE2EContext
sq.JenkinsHost = server.URL
sq.JenkinsJobs = []string{"foo"}
sq.WhitelistOverride = "ok-to-merge"
sq.Initialize(config)
sq.EachLoop()
sq.userWhitelist.Insert(whitelistUser)
obj := github_util.TestObject(config, test.issue, test.pr, test.commits, test.events)
sq.Munge(obj)
done := make(chan bool, 1)
go func(done chan bool) {
for {
reason := sq.prStatus["1"].Reason
for _, r := range test.reasons {
if r == reason {
done <- true
return
}
}
time.Sleep(1 * time.Millisecond)
}
}(done)
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatalf("test:%d timed out waiting expected reason=%v but got %q", testNum, test.reasons, sq.prStatus["1"].Reason)
}
server.Close()
}
}