func dropTestDB(t *testing.T) {
session, _, err := db.GetGlobalSessionFactory().GetSession()
testutil.HandleTestingErr(err, t, "Error opening database session")
defer session.Close()
testutil.HandleTestingErr(session.DB(testConfig.Db).DropDatabase(), t, "Error "+
"dropping test database")
}
// helper for getting the correct db
func getSessionAndDB() (*mgo.Session, *mgo.Database, error) {
session, _, err := db.GetGlobalSessionFactory().GetSession()
if err != nil {
return nil, nil, err
}
return session, session.DB(TaskLogDB), nil
}
func (self *taskHistoryIterator) GetDistinctTestNames(numCommits int) ([]string, error) {
session, db, err := db.GetGlobalSessionFactory().GetSession()
defer session.Close()
pipeline := db.C(TasksCollection).Pipe(
[]bson.M{
{
"$match": bson.M{
TaskBuildVariantKey: bson.M{"$in": self.BuildVariants},
TaskDisplayNameKey: self.TaskName,
},
},
bson.M{"$sort": bson.D{{TaskRevisionOrderNumberKey, -1}}},
bson.M{"$limit": numCommits},
bson.M{"$unwind": fmt.Sprintf("$%v", TaskTestResultsKey)},
bson.M{"$group": bson.M{"_id": fmt.Sprintf("$%v.%v", TaskTestResultsKey, TestResultTestFileKey)}},
},
)
var output []bson.M
err = pipeline.All(&output)
if err != nil {
return nil, err
}
names := make([]string, 0)
for _, doc := range output {
names = append(names, doc["_id"].(string))
}
return names, nil
}
// finds a bookkeeping entry matching the specified interface
func findOneTaskBk(matcher interface{}, selector interface{}) (*TaskBookkeeping, error) {
// establish a database connection
session, db, err := db.GetGlobalSessionFactory().GetSession()
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error establishing database connection: %v", err)
return nil, err
}
// make sure the function is closed when the function exits
defer session.Close()
// query for the bookkeeping entry
taskBk := &TaskBookkeeping{}
err = db.C(TaskBookkeepingCollection).Find(matcher).Select(selector).One(taskBk)
// no entry was found
if err == mgo.ErrNotFound {
return nil, nil
}
// failure
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Unexpected error retrieving task bookkeeping entry from database: %v",
err)
return nil, err
}
// success
return taskBk, nil
}
func cleanUpLogDB() error {
session, _, err := db.GetGlobalSessionFactory().GetSession()
if err != nil {
return err
}
defer session.Close()
_, err = session.DB(TaskLogDB).C(TaskLogCollection).RemoveAll(bson.M{})
return err
}
// upsert a single bookkeeping entry
func upsertOneTaskBk(matcher interface{}, update interface{}) error {
// establish a database connection
session, db, err := db.GetGlobalSessionFactory().GetSession()
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error establishing database connection: %v", err)
return err
}
// make sure the session is closed when the function exits
defer session.Close()
// update the bookkeeping entry
_, err = db.C(TaskBookkeepingCollection).Upsert(matcher, update)
return err
}
func SetupAPITestData(taskDisplayName string, isPatch bool, t *testing.T) (*model.Task, *build.Build, error) {
//ignore errs here because the ns might just not exist.
testutil.HandleTestingErr(
db.ClearCollections(model.TasksCollection, build.Collection,
host.Collection, version.Collection, patch.Collection),
t, "Failed to clear test collections")
testHost := &host.Host{
Id: "testHost",
Host: "testHost",
RunningTask: "testTaskId",
StartedBy: evergreen.User,
}
testutil.HandleTestingErr(testHost.Insert(), t, "failed to insert host")
task := &model.Task{
Id: "testTaskId",
BuildId: "testBuildId",
DistroId: "rhel55",
BuildVariant: "linux-64",
Project: "mongodb-mongo-master",
DisplayName: taskDisplayName,
HostId: "testHost",
Version: "testVersionId",
Secret: "testTaskSecret",
Status: evergreen.TaskDispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
if isPatch {
task.Requester = evergreen.PatchVersionRequester
}
testutil.HandleTestingErr(task.Insert(), t, "failed to insert task")
version := &version.Version{Id: "testVersionId", BuildIds: []string{task.BuildId}}
testutil.HandleTestingErr(version.Insert(), t, "failed to insert version %v")
if isPatch {
modulePatchContent, err := ioutil.ReadFile("testdata/testmodule.patch")
testutil.HandleTestingErr(err, t, "failed to read test module patch file %v")
patch := &patch.Patch{
Status: evergreen.PatchCreated,
Version: version.Id,
Patches: []patch.ModulePatch{
{
ModuleName: "enterprise",
Githash: "c2d7ce942a96d7dacd27c55b257e3f2774e04abf",
PatchSet: patch.PatchSet{Patch: string(modulePatchContent)},
},
},
}
testutil.HandleTestingErr(patch.Insert(), t, "failed to insert patch %v")
}
session, _, err := db.GetGlobalSessionFactory().GetSession()
testutil.HandleTestingErr(err, t, "couldn't get db session!")
//Remove any logs for our test task from previous runs.
_, err = session.DB(model.TaskLogDB).C(model.TaskLogCollection).RemoveAll(bson.M{"t_id": task.Id})
testutil.HandleTestingErr(err, t, "failed to remove logs")
build := &build.Build{Id: "testBuildId", Tasks: []build.TaskCache{build.NewTaskCache(task.Id, task.DisplayName, true)}}
testutil.HandleTestingErr(build.Insert(), t, "failed to insert build %v")
return task, build, nil
}
// Returns tasks grouped by their versions, and sorted with the most
// recent first (i.e. descending commit order number).
func (iter *taskHistoryIterator) GetChunk(v *version.Version, numBefore, numAfter int, include bool) (TaskHistoryChunk, error) {
session, database, err := db.GetGlobalSessionFactory().GetSession()
defer session.Close()
chunk := TaskHistoryChunk{
Tasks: []bson.M{},
Versions: []version.Version{},
FailedTests: map[string][]TestResult{},
}
versionsBefore, exhausted, err := iter.findAllVersions(v, numBefore, true, include)
if err != nil {
return chunk, err
}
chunk.Exhausted.Before = exhausted
versionsAfter, exhausted, err := iter.findAllVersions(v, numAfter, false, false)
if err != nil {
return chunk, err
}
chunk.Exhausted.After = exhausted
versions := append(versionsAfter, versionsBefore...)
if len(versions) == 0 {
return chunk, nil
}
chunk.Versions = versions
// versionStartBoundary is the most recent version (i.e. newest) that
// should be included in the results.
//
// versionEndBoundary is the least recent version (i.e. oldest) that
// should be included in the results.
versionStartBoundary, versionEndBoundary := versions[0], versions[len(versions)-1]
pipeline := database.C(TasksCollection).Pipe(
[]bson.M{
{"$match": bson.M{
TaskRequesterKey: evergreen.RepotrackerVersionRequester,
TaskProjectKey: iter.ProjectName,
TaskDisplayNameKey: iter.TaskName,
TaskBuildVariantKey: bson.M{"$in": iter.BuildVariants},
TaskRevisionOrderNumberKey: bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
"$lte": versionStartBoundary.RevisionOrderNumber,
},
}},
{"$project": bson.M{
TaskIdKey: 1,
TaskStatusKey: 1,
TaskDetailsKey: 1,
TaskActivatedKey: 1,
TaskTimeTakenKey: 1,
TaskBuildVariantKey: 1,
TaskRevisionKey: 1,
TaskRevisionOrderNumberKey: 1,
}},
{"$group": bson.M{
"_id": fmt.Sprintf("$%v", TaskRevisionKey),
"order": bson.M{"$first": fmt.Sprintf("$%v", TaskRevisionOrderNumberKey)},
"tasks": bson.M{
"$push": bson.M{
TaskIdKey: fmt.Sprintf("$%v", TaskIdKey),
TaskStatusKey: fmt.Sprintf("$%v", TaskStatusKey),
TaskDetailsKey: fmt.Sprintf("$%v", TaskDetailsKey),
TaskActivatedKey: fmt.Sprintf("$%v", TaskActivatedKey),
TaskTimeTakenKey: fmt.Sprintf("$%v", TaskTimeTakenKey),
TaskBuildVariantKey: fmt.Sprintf("$%v", TaskBuildVariantKey),
},
},
}},
{"$sort": bson.M{TaskRevisionOrderNumberKey: -1}},
},
)
var aggregatedTasks []bson.M
if err := pipeline.All(&aggregatedTasks); err != nil {
return chunk, err
}
chunk.Tasks = aggregatedTasks
failedTests, err := iter.GetFailedTests(pipeline)
if err != nil {
return chunk, err
}
chunk.FailedTests = failedTests
return chunk, nil
}
// Returns versions and tasks grouped by gitspec, newest first (sorted by order number desc)
func (self *buildVariantHistoryIterator) GetItems(beforeCommit *version.Version, numRevisions int) ([]bson.M, []version.Version, error) {
session, dbobj, err := db.GetGlobalSessionFactory().GetSession()
defer session.Close()
if err != nil {
return nil, nil, err
}
var versionQuery db.Q
if beforeCommit != nil {
versionQuery = db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.RevisionOrderNumberKey: bson.M{"$lt": beforeCommit.RevisionOrderNumber},
version.IdentifierKey: self.ProjectName,
version.BuildVariantsKey: bson.M{
"$elemMatch": bson.M{
version.BuildStatusVariantKey: self.BuildVariantInVersion,
},
},
})
} else {
versionQuery = db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.IdentifierKey: self.ProjectName,
version.BuildVariantsKey: bson.M{
"$elemMatch": bson.M{
version.BuildStatusVariantKey: self.BuildVariantInVersion,
},
},
})
}
versionQuery = versionQuery.WithFields(
version.IdKey,
version.RevisionOrderNumberKey,
version.RevisionKey,
version.MessageKey,
version.CreateTimeKey,
).Sort([]string{"-" + version.RevisionOrderNumberKey}).Limit(numRevisions)
//Get the next numCommits
versions, err := version.Find(versionQuery)
if err != nil {
return nil, nil, err
}
if len(versions) == 0 {
return nil, []version.Version{}, nil
}
//versionEndBoundary is the *earliest* version which should be included in results
versionEndBoundary := versions[len(versions)-1]
matchFilter := bson.M{
TaskRequesterKey: evergreen.RepotrackerVersionRequester,
TaskBuildVariantKey: self.BuildVariantInTask,
TaskProjectKey: self.ProjectName,
}
if beforeCommit != nil {
matchFilter[TaskRevisionOrderNumberKey] = bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
"$lt": beforeCommit.RevisionOrderNumber,
}
} else {
matchFilter[TaskRevisionOrderNumberKey] = bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
}
}
pipeline := dbobj.C(TasksCollection).Pipe(
[]bson.M{
{"$match": matchFilter},
bson.M{"$sort": bson.D{{TaskRevisionOrderNumberKey, 1}}},
bson.M{
"$group": bson.M{
"_id": "$" + TaskRevisionKey,
"order": bson.M{"$first": "$" + TaskRevisionOrderNumberKey},
"tasks": bson.M{
"$push": bson.M{
"_id": "$" + TaskIdKey,
"status": "$" + TaskStatusKey,
"task_end_details": "$" + TaskDetailsKey,
"activated": "$" + TaskActivatedKey,
"time_taken": "$" + TaskTimeTakenKey,
"display_name": "$" + TaskDisplayNameKey,
},
},
},
},
bson.M{"$sort": bson.M{TaskRevisionOrderNumberKey: -1, TaskDisplayNameKey: 1}},
},
)
var output []bson.M
err = pipeline.All(&output)
if err != nil {
return nil, nil, err
}
return output, versions, nil
}
func setupAPITestData(testConfig *evergreen.Settings, taskDisplayName string,
variant string, patchMode patchTestMode, t *testing.T) (*model.Task, *build.Build, error) {
//ignore errs here because the ns might just not exist.
clearDataMsg := "Failed to clear test data collection"
testCollections := []string{
model.TasksCollection, build.Collection, host.Collection,
distro.Collection, version.Collection, patch.Collection,
model.PushlogCollection, model.ProjectVarsCollection, model.TaskQueuesCollection,
manifest.Collection, model.ProjectRefCollection}
testutil.HandleTestingErr(dbutil.ClearCollections(testCollections...), t, clearDataMsg)
projectVars := &model.ProjectVars{
Id: "evergreen-ci-render",
Vars: map[string]string{
"aws_key": testConfig.Providers.AWS.Id,
"aws_secret": testConfig.Providers.AWS.Secret,
"fetch_key": "fetch_expansion_value",
},
}
_, err := projectVars.Upsert()
testutil.HandleTestingErr(err, t, clearDataMsg)
taskOne := &model.Task{
Id: "testTaskId",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: "evergreen-ci-render",
DisplayName: taskDisplayName,
HostId: "testHost",
Secret: "testTaskSecret",
Version: "testVersionId",
Status: evergreen.TaskDispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
taskTwo := &model.Task{
Id: "testTaskIdTwo",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: "evergreen-ci-render",
DisplayName: taskDisplayName,
HostId: "",
Secret: "testTaskSecret",
Activated: true,
Version: "testVersionId",
Status: evergreen.TaskUndispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
if patchMode != NoPatch {
taskOne.Requester = evergreen.PatchVersionRequester
}
testutil.HandleTestingErr(taskOne.Insert(), t, "failed to insert taskOne")
testutil.HandleTestingErr(taskTwo.Insert(), t, "failed to insert taskTwo")
// set up task queue for task end tests
taskQueue := &model.TaskQueue{
Distro: "test-distro-one",
Queue: []model.TaskQueueItem{
model.TaskQueueItem{
Id: "testTaskIdTwo",
DisplayName: taskDisplayName,
},
},
}
testutil.HandleTestingErr(taskQueue.Save(), t, "failed to insert taskqueue")
workDir, err := ioutil.TempDir("", "agent_test_")
testutil.HandleTestingErr(err, t, "failed to create working directory")
host := &host.Host{
Id: "testHost",
Host: "testHost",
Distro: distro.Distro{
Id: "test-distro-one",
WorkDir: workDir,
Expansions: []distro.Expansion{{"distro_exp", "DISTRO_EXP"}},
},
RunningTask: "testTaskId",
StartedBy: evergreen.User,
AgentRevision: agentRevision,
}
testutil.HandleTestingErr(host.Insert(), t, "failed to insert host")
// read in the project configuration
projectFile := "testdata/config_test_plugin/project/evergreen-ci-render.yml"
projectConfig, err := ioutil.ReadFile(projectFile)
testutil.HandleTestingErr(err, t, "failed to read project config")
projectRef := &model.ProjectRef{
Identifier: "evergreen-ci-render",
Owner: "evergreen-ci",
Repo: "render",
RepoKind: "github",
Branch: "master",
Enabled: true,
BatchTime: 180,
}
testutil.HandleTestingErr(projectRef.Insert(), t, "failed to insert projectRef")
err = testutil.CreateTestLocalConfig(testConfig, "evergreen-ci-render", "testdata/config_test_plugin/project/evergreen-ci-render.yml")
testutil.HandleTestingErr(err, t, "failed to marshall project config")
// unmarshall the project configuration into a struct
project := &model.Project{}
testutil.HandleTestingErr(yaml.Unmarshal(projectConfig, project), t, "failed to unmarshal project config")
// now then marshall the project YAML for storage
projectYamlBytes, err := yaml.Marshal(project)
testutil.HandleTestingErr(err, t, "failed to marshall project config")
// insert the version document
v := &version.Version{
Id: "testVersionId",
BuildIds: []string{taskOne.BuildId},
Config: string(projectYamlBytes),
}
testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
if patchMode != NoPatch {
mainPatchContent, err := ioutil.ReadFile("testdata/test.patch")
testutil.HandleTestingErr(err, t, "failed to read test patch file")
modulePatchContent, err := ioutil.ReadFile("testdata/testmodule.patch")
testutil.HandleTestingErr(err, t, "failed to read test module patch file")
ptch := &patch.Patch{
Status: evergreen.PatchCreated,
Version: v.Id,
}
if patchMode == InlinePatch {
ptch.Patches = []patch.ModulePatch{
{
ModuleName: "",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{Patch: string(mainPatchContent)},
},
{
ModuleName: "recursive",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{Patch: string(modulePatchContent)},
},
}
} else {
p1Id, p2Id := bson.NewObjectId().Hex(), bson.NewObjectId().Hex()
So(dbutil.WriteGridFile(patch.GridFSPrefix, p1Id, strings.NewReader(string(mainPatchContent))), ShouldBeNil)
So(dbutil.WriteGridFile(patch.GridFSPrefix, p2Id, strings.NewReader(string(modulePatchContent))), ShouldBeNil)
ptch.Patches = []patch.ModulePatch{
{
ModuleName: "",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{PatchFileId: p1Id},
},
{
ModuleName: "recursive",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{PatchFileId: p2Id},
},
}
}
testutil.HandleTestingErr(ptch.Insert(), t, "failed to insert patch")
}
session, _, err := dbutil.GetGlobalSessionFactory().GetSession()
testutil.HandleTestingErr(err, t, "couldn't get db session!")
// Remove any logs for our test task from previous runs.
_, err = session.DB(model.TaskLogDB).C(model.TaskLogCollection).
RemoveAll(bson.M{"t_id": bson.M{"$in": []string{taskOne.Id, taskTwo.Id}}})
testutil.HandleTestingErr(err, t, "failed to remove logs")
build := &build.Build{
Id: "testBuildId",
Tasks: []build.TaskCache{
build.NewTaskCache(taskOne.Id, taskOne.DisplayName, true),
build.NewTaskCache(taskTwo.Id, taskTwo.DisplayName, true),
},
Version: "testVersionId",
}
testutil.HandleTestingErr(build.Insert(), t, "failed to insert build")
return taskOne, build, nil
}
func setupAPITestData(testConfig *evergreen.Settings, taskDisplayName string,
variant string, projectFile string, patchMode patchTestMode, t *testing.T) (*model.Task, *build.Build, error) {
// Ignore errs here because the ns might just not exist.
clearDataMsg := "Failed to clear test data collection"
testCollections := []string{
model.TasksCollection, build.Collection, host.Collection,
distro.Collection, version.Collection, patch.Collection,
model.PushlogCollection, model.ProjectVarsCollection, model.TaskQueuesCollection,
manifest.Collection, model.ProjectRefCollection}
testutil.HandleTestingErr(dbutil.ClearCollections(testCollections...), t, clearDataMsg)
// Read in the project configuration
projectConfig, err := ioutil.ReadFile(projectFile)
testutil.HandleTestingErr(err, t, "failed to read project config")
// Unmarshall the project configuration into a struct
project := &model.Project{}
testutil.HandleTestingErr(yaml.Unmarshal(projectConfig, project), t, "failed to unmarshal project config")
// Marshall the project YAML for storage
projectYamlBytes, err := yaml.Marshal(project)
testutil.HandleTestingErr(err, t, "failed to marshall project config")
// Create the ref for the project
projectRef := &model.ProjectRef{
Identifier: project.DisplayName,
Owner: project.Owner,
Repo: project.Repo,
RepoKind: project.RepoKind,
Branch: project.Branch,
Enabled: project.Enabled,
BatchTime: project.BatchTime,
LocalConfig: string(projectConfig),
}
testutil.HandleTestingErr(projectRef.Insert(), t, "failed to insert projectRef")
// Save the project variables
projectVars := &model.ProjectVars{
Id: project.DisplayName,
Vars: map[string]string{
"aws_key": testConfig.Providers.AWS.Id,
"aws_secret": testConfig.Providers.AWS.Secret,
"fetch_key": "fetch_expansion_value",
},
}
_, err = projectVars.Upsert()
testutil.HandleTestingErr(err, t, clearDataMsg)
// Create and insert two tasks
taskOne := &model.Task{
Id: "testTaskId",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: project.DisplayName,
DisplayName: taskDisplayName,
HostId: "testHost",
Secret: "testTaskSecret",
Version: "testVersionId",
Status: evergreen.TaskDispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
if patchMode != NoPatch {
taskOne.Requester = evergreen.PatchVersionRequester
}
testutil.HandleTestingErr(taskOne.Insert(), t, "failed to insert taskOne")
taskTwo := &model.Task{
Id: "testTaskIdTwo",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: project.DisplayName,
DisplayName: taskDisplayName,
HostId: "",
Secret: "testTaskSecret",
Version: "testVersionId",
Status: evergreen.TaskUndispatched,
Requester: evergreen.RepotrackerVersionRequester,
Activated: true,
}
testutil.HandleTestingErr(taskTwo.Insert(), t, "failed to insert taskTwo")
// Set up a task queue for task end tests
taskQueue := &model.TaskQueue{
Distro: "test-distro-one",
Queue: []model.TaskQueueItem{
model.TaskQueueItem{
Id: "testTaskIdTwo",
DisplayName: taskDisplayName,
},
},
}
testutil.HandleTestingErr(taskQueue.Save(), t, "failed to insert taskqueue")
// Insert the version document
v := &version.Version{
Id: "testVersionId",
BuildIds: []string{taskOne.BuildId},
Config: string(projectYamlBytes),
}
testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
// Insert the build that contains the tasks
build := &build.Build{
Id: "testBuildId",
Tasks: []build.TaskCache{
build.NewTaskCache(taskOne.Id, taskOne.DisplayName, true),
build.NewTaskCache(taskTwo.Id, taskTwo.DisplayName, true),
},
Version: v.Id,
}
testutil.HandleTestingErr(build.Insert(), t, "failed to insert build")
workDir, err := ioutil.TempDir("", "agent_test_")
testutil.HandleTestingErr(err, t, "failed to create working directory")
// Insert the host info for running the tests
host := &host.Host{
Id: "testHost",
Host: "testHost",
Distro: distro.Distro{
Id: "test-distro-one",
WorkDir: workDir,
Expansions: []distro.Expansion{{"distro_exp", "DISTRO_EXP"}},
},
RunningTask: taskOne.Id,
StartedBy: evergreen.User,
AgentRevision: agentRevision,
}
testutil.HandleTestingErr(host.Insert(), t, "failed to insert host")
session, _, err := dbutil.GetGlobalSessionFactory().GetSession()
testutil.HandleTestingErr(err, t, "couldn't get db session!")
// Remove any logs for our test task from previous runs.
_, err = session.DB(model.TaskLogDB).C(model.TaskLogCollection).
RemoveAll(bson.M{"t_id": bson.M{"$in": []string{taskOne.Id, taskTwo.Id}}})
testutil.HandleTestingErr(err, t, "failed to remove logs")
return taskOne, build, nil
}