// createOneTask is a helper to create a single task. func createOneTask(id string, buildVarTask BuildVariantTask, project *Project, buildVariant *BuildVariant, b *build.Build, v *version.Version) *Task { return &Task{ Id: id, Secret: util.RandomString(), DisplayName: buildVarTask.Name, BuildId: b.Id, BuildVariant: buildVariant.Name, CreateTime: b.CreateTime, PushTime: b.PushTime, ScheduledTime: ZeroTime, StartTime: ZeroTime, // Certain time fields must be initialized FinishTime: ZeroTime, // to our own ZeroTime value (which is DispatchTime: ZeroTime, // Unix epoch 0, not Go's time.Time{}) LastHeartbeat: ZeroTime, Status: evergreen.TaskUndispatched, Activated: b.Activated, RevisionOrderNumber: v.RevisionOrderNumber, Requester: v.Requester, Version: v.Id, Revision: v.Revision, Project: project.Identifier, Priority: buildVarTask.Priority, } }
// Reset sets the task state to be activated, with a new secret, // undispatched status and zero time on Start, Scheduled, Dispatch and FinishTime func ResetTasks(taskIds []string) error { reset := bson.M{ "$set": bson.M{ ActivatedKey: true, SecretKey: util.RandomString(), StatusKey: evergreen.TaskUndispatched, DispatchTimeKey: util.ZeroTime, StartTimeKey: util.ZeroTime, ScheduledTimeKey: util.ZeroTime, FinishTimeKey: util.ZeroTime, TestResultsKey: []TestResult{}, }, "$unset": bson.M{ DetailsKey: "", }, } _, err := UpdateAll( bson.M{ IdKey: bson.M{"$in": taskIds}, }, reset, ) return err }
func (t *Task) reset() error { if err := t.Archive(); err != nil { return fmt.Errorf("Can't restart task because it can't be archived: %v", err) } reset := bson.M{ "$set": bson.M{ TaskActivatedKey: true, TaskSecretKey: util.RandomString(), TaskStatusKey: evergreen.TaskUndispatched, TaskDispatchTimeKey: ZeroTime, TaskStartTimeKey: ZeroTime, TaskScheduledTimeKey: ZeroTime, TaskFinishTimeKey: ZeroTime, TaskTestResultsKey: []TestResult{}, }, "$unset": bson.M{ TaskDetailsKey: "", }, } err := UpdateOneTask(bson.M{TaskIdKey: t.Id}, reset) if err != nil { return err } // update the cached version of the task, in its build document if err = build.ResetCachedTask(t.BuildId, t.Id); err != nil { return err } return t.UpdateBuildStatus() }
// Reset sets the task state to be activated, with a new secret, // undispatched status and zero time on Start, Scheduled, Dispatch and FinishTime func (t *Task) Reset() error { t.Activated = true t.Secret = util.RandomString() t.DispatchTime = util.ZeroTime t.StartTime = util.ZeroTime t.ScheduledTime = util.ZeroTime t.FinishTime = util.ZeroTime t.TestResults = []TestResult{} reset := bson.M{ "$set": bson.M{ ActivatedKey: true, SecretKey: t.Secret, StatusKey: evergreen.TaskUndispatched, DispatchTimeKey: util.ZeroTime, StartTimeKey: util.ZeroTime, ScheduledTimeKey: util.ZeroTime, FinishTimeKey: util.ZeroTime, TestResultsKey: []TestResult{}, }, "$unset": bson.M{ DetailsKey: "", }, } return UpdateOne( bson.M{ IdKey: t.Id, }, reset, ) }
func (staticMgr *MockCloudManager) SpawnInstance(distro *distro.Distro, owner string, userHost bool) (*host.Host, error) { return &host.Host{ Id: util.RandomString(), Distro: *distro, StartedBy: owner, }, nil }
func (staticMgr *MockCloudManager) SpawnInstance(distro *distro.Distro, hostOpts cloud.HostOptions) (*host.Host, error) { return &host.Host{ Id: util.RandomString(), Distro: *distro, StartedBy: hostOpts.UserName, }, nil }
// NewGithubUserManager initializes a GithubUserManager with a Salt as randomly generated string used in Github // authentication func NewGithubUserManager(g *evergreen.GithubAuthConfig) (*GithubUserManager, error) { if g.ClientId == "" { return nil, fmt.Errorf("no client id for config") } if g.ClientSecret == "" { return nil, fmt.Errorf("no client secret for config given") } return &GithubUserManager{g.ClientId, g.ClientSecret, g.Users, g.Organization, util.RandomString()}, nil }
func (uis *UIServer) newAPIKey(w http.ResponseWriter, r *http.Request) { currentUser := MustHaveUser(r) newKey := util.RandomString() if err := model.SetUserAPIKey(currentUser.Id, newKey); err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed saving key: %v", err)) return } uis.WriteJSON(w, http.StatusOK, struct { Key string `json:"key"` }{newKey}) }
func TestPutS3File(t *testing.T) { testutil.ConfigureIntegrationTest(t, testConfig, "TestPutS3File") Convey("When given a file to copy to S3...", t, func() { Convey("a valid source file with a long key should return an error ", func() { //Make a test file with some random content. tempfile, err := ioutil.TempFile("", "randomString") So(err, ShouldEqual, nil) randStr := util.RandomString() _, err = tempfile.Write([]byte(randStr)) So(err, ShouldEqual, nil) tempfile.Close() // put the test file on S3 auth := &aws.Auth{ AccessKey: testConfig.Providers.AWS.Id, SecretKey: testConfig.Providers.AWS.Secret, } longURLKey := sourceURL + strings.Repeat("suffix", 300) err = PutS3File(auth, tempfile.Name(), longURLKey, "application/x-tar", "public-read") So(err, ShouldNotEqual, nil) }) Convey("a valid source file with a valid key should return no errors", func() { //Make a test file with some random content. tempfile, err := ioutil.TempFile("", "randomString") So(err, ShouldEqual, nil) randStr := util.RandomString() _, err = tempfile.Write([]byte(randStr)) So(err, ShouldEqual, nil) tempfile.Close() // put the test file on S3 auth := &aws.Auth{ AccessKey: testConfig.Providers.AWS.Id, SecretKey: testConfig.Providers.AWS.Secret, } err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read") So(err, ShouldEqual, nil) }) }) }
// GitApplyNumstat attempts to apply a given patch; it returns the patch's bytes // if it is successful func GitApplyNumstat(patch string) (*bytes.Buffer, error) { handle, err := ioutil.TempFile("", util.RandomString()) if err != nil { return nil, fmt.Errorf("Unable to create local patch file") } // convert the patch to bytes buf := []byte(patch) buffer := bytes.NewBuffer(buf) for { // read a chunk n, err := buffer.Read(buf) if err != nil && err != io.EOF { return nil, fmt.Errorf("Unable to read supplied patch file") } if n == 0 { break } // write a chunk if _, err := handle.Write(buf[:n]); err != nil { return nil, fmt.Errorf("Unable to read supplied patch file") } } // pseudo-validate the patch set by attempting to get a summary var summaryBuffer bytes.Buffer cmd := exec.Command("git", "apply", "--numstat", handle.Name()) cmd.Stdout = &summaryBuffer cmd.Stderr = &summaryBuffer cmd.Dir = filepath.Dir(handle.Name()) // this should never happen if patch is initially validated if err := cmd.Start(); err != nil { return nil, fmt.Errorf("Error validating patch: 424 - %v (%v)", summaryBuffer.String(), err) } // this should never happen if patch is initially validated if err := cmd.Wait(); err != nil { return nil, fmt.Errorf("Error waiting on patch: 562 - %v (%v)", summaryBuffer.String(), err) } return &summaryBuffer, nil }
// GetOrCreateUser fetches a user with the given userId and returns it. If no document exists for // that userId, inserts it along with the provided display name and email. func GetOrCreateUser(userId, displayName, email string) (*user.DBUser, error) { u := &user.DBUser{} _, err := db.FindAndModify(user.Collection, bson.M{user.IdKey: userId}, nil, mgo.Change{ Update: bson.M{ "$set": bson.M{ user.DispNameKey: displayName, user.EmailAddressKey: email, }, "$setOnInsert": bson.M{ user.APIKeyKey: util.RandomString(), }, }, ReturnNew: true, Upsert: true, }, u) if err != nil { return nil, err } return u, nil }
// RestartVersion restarts completed tasks associated with a given versionId. // If abortInProgress is true, it also sets the abort flag on any in-progress tasks. func RestartVersion(versionId string, taskIds []string, abortInProgress bool, caller string) error { // restart all the 'not in-progress' tasks for the version allTasks, err := FindAllTasks( bson.M{ TaskIdKey: bson.M{"$in": taskIds}, TaskVersionKey: versionId, TaskDispatchTimeKey: bson.M{"$ne": ZeroTime}, TaskStatusKey: bson.M{ "$in": []string{ evergreen.TaskSucceeded, evergreen.TaskFailed, }, }, }, db.NoProjection, db.NoSort, db.NoSkip, db.NoLimit, ) if err != nil && err != mgo.ErrNotFound { return err } // archive all the tasks for _, t := range allTasks { if err := t.Archive(); err != nil { return fmt.Errorf("failed to archive task: %v", err) } } // Set all the task fields to indicate restarted _, err = UpdateAllTasks( bson.M{TaskIdKey: bson.M{"$in": taskIds}}, bson.M{ "$set": bson.M{ TaskActivatedKey: true, TaskSecretKey: util.RandomString(), TaskStatusKey: evergreen.TaskUndispatched, TaskDispatchTimeKey: ZeroTime, TaskStartTimeKey: ZeroTime, TaskScheduledTimeKey: ZeroTime, TaskFinishTimeKey: ZeroTime, TaskTestResultsKey: []TestResult{}, }, "$unset": bson.M{ TaskDetailsKey: "", }, }) if err != nil { return err } // TODO figure out a way to coalesce updates for task cache for the same build, so we // only need to do one update per-build instead of one per-task here. // Doesn't seem to be possible as-is because $ can only apply to one array element matched per // document. buildIdSet := map[string]bool{} for _, t := range allTasks { buildIdSet[t.BuildId] = true err = build.ResetCachedTask(t.BuildId, t.Id) if err != nil { return err } } // reset the build statuses, once per build buildIdList := make([]string, 0, len(buildIdSet)) for k, _ := range buildIdSet { buildIdList = append(buildIdList, k) } // Set the build status for all the builds containing the tasks that we touched _, err = build.UpdateAllBuilds( bson.M{build.IdKey: bson.M{"$in": buildIdList}}, bson.M{"$set": bson.M{build.StatusKey: evergreen.BuildStarted}}, ) if abortInProgress { // abort in-progress tasks in this build _, err = UpdateAllTasks( bson.M{ TaskVersionKey: versionId, TaskIdKey: bson.M{"$in": taskIds}, TaskStatusKey: bson.M{"$in": evergreen.AbortableStatuses}, }, bson.M{"$set": bson.M{TaskAbortedKey: true}}, ) if err != nil { return err } } // update activation for all the builds for _, b := range buildIdList { err := build.UpdateActivation(b, true, caller) if err != nil { return err } } return nil }
func TestS3Copy(t *testing.T) { testutil.ConfigureIntegrationTest(t, testConfig, "TestS3Copy") Convey("When given a source and destination URL to copy from/to...", t, func() { Convey("a valid source file should be copied to the valid destination", func() { //Make a test file with some random content. tempfile, err := ioutil.TempFile("", "randomString") So(err, ShouldEqual, nil) randStr := util.RandomString() _, err = tempfile.Write([]byte(randStr)) So(err, ShouldEqual, nil) tempfile.Close() // Put the test file on S3 auth := &aws.Auth{ AccessKey: testConfig.Providers.AWS.Id, SecretKey: testConfig.Providers.AWS.Secret, } err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read") So(err, ShouldEqual, nil) // Copy the test file over to another location err = CopyS3File(auth, sourceURL, destUrl, "public-read") So(err, ShouldEqual, nil) // Ensure that the file was actually copied rdr, err := GetS3File(auth, destUrl) defer rdr.Close() So(err, ShouldEqual, nil) fileContents, err := ioutil.ReadAll(rdr) So(err, ShouldEqual, nil) So(string(fileContents), ShouldEqual, randStr) }) Convey("a valid source file with a long key should return an error "+ "even if sent to a valid destination", func() { //Make a test file with some random content. tempfile, err := ioutil.TempFile("", "randomString") So(err, ShouldEqual, nil) randStr := util.RandomString() _, err = tempfile.Write([]byte(randStr)) So(err, ShouldEqual, nil) tempfile.Close() // put the test file on S3 auth := &aws.Auth{ AccessKey: testConfig.Providers.AWS.Id, SecretKey: testConfig.Providers.AWS.Secret, } longURLKey := sourceURL + strings.Repeat("suffix", 300) err = PutS3File(auth, tempfile.Name(), longURLKey, "application/x-tar", "public-read") So(err, ShouldNotEqual, nil) }) Convey("a valid source file copied to a destination with too long a "+ "key name should return an error", func() { //Make a test file with some random content. tempfile, err := ioutil.TempFile("", "randomString") So(err, ShouldEqual, nil) randStr := util.RandomString() _, err = tempfile.Write([]byte(randStr)) So(err, ShouldEqual, nil) tempfile.Close() // put the test file on S3 auth := &aws.Auth{ AccessKey: testConfig.Providers.AWS.Id, SecretKey: testConfig.Providers.AWS.Secret, } err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read") So(err, ShouldEqual, nil) longURLKey := sourceURL + strings.Repeat("suffix", 300) // Attempt to copy the test file over to another location err = CopyS3File(auth, sourceURL, longURLKey, "public-read") So(err, ShouldNotEqual, nil) // Ensure that the file was not copied rdr, err := GetS3File(auth, longURLKey) So(rdr, ShouldEqual, nil) So(err, ShouldNotEqual, nil) }) }) }
func TestS3PutAndGet(t *testing.T) { testutil.HandleTestingErr( db.ClearCollections(task.Collection, artifact.Collection), t, "error clearing test collections") conf := evergreen.TestConfig() testutil.ConfigureIntegrationTest(t, conf, "TestS3PutAndGet") Convey("When putting to and retrieving from an s3 bucket", t, func() { var putCmd *S3PutCommand var getCmd *S3GetCommand testDataDir := "testdata" remoteFile := "remote_mci_put_test.tgz" bucket := "mci-test-uploads" permissions := "private" contentType := "application/x-tar" displayName := "testfile" // create the local directory to be tarred localDirToTar := filepath.Join(testDataDir, "put_test") localFileToTar := filepath.Join(localDirToTar, "put_test_file.txt") testutil.HandleTestingErr(os.RemoveAll(localDirToTar), t, "Error removing"+ " directory") testutil.HandleTestingErr(os.MkdirAll(localDirToTar, 0755), t, "Error creating directory") randStr := util.RandomString() So(ioutil.WriteFile(localFileToTar, []byte(randStr), 0755), ShouldBeNil) // tar it tarCmd := &command.LocalCommand{ CmdString: "tar czf put_test.tgz put_test", WorkingDirectory: testDataDir, Stdout: ioutil.Discard, Stderr: ioutil.Discard, } testutil.HandleTestingErr(tarCmd.Run(), t, "Error tarring directories") tarballSource := filepath.Join(testDataDir, "put_test.tgz") // remove the untarred version testutil.HandleTestingErr(os.RemoveAll(localDirToTar), t, "Error removing directories") Convey("the file retrieved should be the exact same as the file put", func() { // load params into the put command putCmd = &S3PutCommand{} putParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "local_file": tarballSource, "remote_file": remoteFile, "bucket": bucket, "permissions": permissions, "content_type": contentType, "display_name": displayName, } So(putCmd.ParseParams(putParams), ShouldBeNil) So(putCmd.Put(), ShouldBeNil) // next, get the file, untarring it getCmd = &S3GetCommand{} getParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "remote_file": remoteFile, "bucket": bucket, "extract_to": testDataDir, "display_name": displayName, } So(getCmd.ParseParams(getParams), ShouldBeNil) So(getCmd.Get(), ShouldBeNil) // read in the file that was pulled down fileContents, err := ioutil.ReadFile(localFileToTar) So(err, ShouldBeNil) So(string(fileContents), ShouldEqual, randStr) // now, get the tarball without untarring it getCmd = &S3GetCommand{} localDlTarget := filepath.Join(testDataDir, "put_test_dl.tgz") getParams = map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "remote_file": remoteFile, "bucket": bucket, "local_file": localDlTarget, "display_name": displayName, } So(getCmd.ParseParams(getParams), ShouldBeNil) So(getCmd.Get(), ShouldBeNil) exists, err := util.FileExists(localDlTarget) So(err, ShouldBeNil) So(exists, ShouldBeTrue) }) Convey("the put command should always run if there is no variants filter", func() { // load params into the put command putCmd = &S3PutCommand{} putParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "local_file": tarballSource, "remote_file": remoteFile, "bucket": bucket, "permissions": permissions, "content_type": contentType, "display_name": displayName, } So(putCmd.ParseParams(putParams), ShouldBeNil) So(putCmd.shouldRunForVariant("linux-64"), ShouldBeTrue) }) Convey("put cmd with variants filter should only run if variant is in list", func() { // load params into the put command putCmd = &S3PutCommand{} putParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "local_file": tarballSource, "remote_file": remoteFile, "bucket": bucket, "permissions": permissions, "content_type": contentType, "display_name": displayName, "build_variants": []string{"linux-64", "windows-64"}, } So(putCmd.ParseParams(putParams), ShouldBeNil) So(putCmd.shouldRunForVariant("linux-64"), ShouldBeTrue) So(putCmd.shouldRunForVariant("osx-108"), ShouldBeFalse) }) Convey("put cmd with 'optional' and missing file should not throw an error", func() { // load params into the put command putCmd = &S3PutCommand{} putParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "optional": true, "local_file": "this_file_does_not_exist.txt", "remote_file": "remote_file", "bucket": "test_bucket", "permissions": "private", "content_type": "text/plain", } So(putCmd.ParseParams(putParams), ShouldBeNil) server, err := apiserver.CreateTestServer(conf, nil, plugin.APIPlugins, false) httpCom := plugintest.TestAgentCommunicator("testTask", "taskSecret", server.URL) pluginCom := &agent.TaskJSONCommunicator{"s3", httpCom} So(err, ShouldBeNil) err = putCmd.Execute(&plugintest.MockLogger{}, pluginCom, &model.TaskConfig{nil, nil, nil, nil, &model.BuildVariant{Name: "linux"}, &command.Expansions{}, "."}, make(chan bool)) So(err, ShouldBeNil) }) }) }
func TestS3PutAndGetMultiFile(t *testing.T) { testutil.HandleTestingErr( db.ClearCollections(task.Collection, artifact.Collection), t, "error clearing test collections") conf := evergreen.TestConfig() testutil.ConfigureIntegrationTest(t, conf, "TestS3PutAndGet") Convey("When putting to and retrieving from an s3 bucket", t, func() { testDataDir := "testdata" bucket := "mci-test-uploads" permissions := "private" contentType := "application/x-tar" remoteFilePrefix := "remote_mci_put_test-" displayNamePrefix := "testfile-" // create the local directory localDir := filepath.Join(testDataDir, "put_test") localFilePrefix := filepath.Join(localDir, "put_test_file") defer func() { testutil.HandleTestingErr(os.RemoveAll(testDataDir), t, "error removing test dir") }() testutil.HandleTestingErr(os.RemoveAll(localDir), t, "Error removing directory") testutil.HandleTestingErr(os.MkdirAll(localDir, 0755), t, "Error creating directory") fileInfos := []multiPutFileInfo{} for i := 0; i < 5; i++ { randStr := util.RandomString() localPath := fmt.Sprintf("%s%d.txt", localFilePrefix, i) localName := filepath.Base(localPath) remoteName := fmt.Sprintf("%s%s", remoteFilePrefix, localName) displayName := fmt.Sprintf("%s%s", displayNamePrefix, localName) So(ioutil.WriteFile(localPath, []byte(randStr), 0755), ShouldBeNil) fileInfo := multiPutFileInfo{ remoteName: remoteName, localPath: localPath, displayName: displayName, data: randStr, } fileInfos = append(fileInfos, fileInfo) } Convey("the files should be put without error", func() { includesFilter := []string{fmt.Sprintf("%s/*.txt", localDir)} // load params into the put command putCmd := &S3PutCommand{} putParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "local_files_include_filter": includesFilter, "remote_file": remoteFilePrefix, "bucket": bucket, "permissions": permissions, "content_type": contentType, "display_name": displayNamePrefix, } So(putCmd.ParseParams(putParams), ShouldBeNil) So(putCmd.Put(), ShouldBeNil) Convey("the files should each be gotten without error", func() { for _, f := range fileInfos { // next, get the file getCmd := &S3GetCommand{} getParams := map[string]interface{}{ "aws_key": conf.Providers.AWS.Id, "aws_secret": conf.Providers.AWS.Secret, "remote_file": f.remoteName, "bucket": bucket, "local_file": f.localPath, "display_name": f.displayName, } So(getCmd.ParseParams(getParams), ShouldBeNil) So(getCmd.Get(), ShouldBeNil) // read in the file that was pulled down fileContents, err := ioutil.ReadFile(f.localPath) So(err, ShouldBeNil) So(string(fileContents), ShouldEqual, f.data) } }) }) }) }