// createOneTask is a helper to create a single task.
func createOneTask(id string, buildVarTask BuildVariantTask, project *Project,
buildVariant *BuildVariant, b *build.Build, v *version.Version) *Task {
return &Task{
Id: id,
Secret: util.RandomString(),
DisplayName: buildVarTask.Name,
BuildId: b.Id,
BuildVariant: buildVariant.Name,
CreateTime: b.CreateTime,
PushTime: b.PushTime,
ScheduledTime: ZeroTime,
StartTime: ZeroTime, // Certain time fields must be initialized
FinishTime: ZeroTime, // to our own ZeroTime value (which is
DispatchTime: ZeroTime, // Unix epoch 0, not Go's time.Time{})
LastHeartbeat: ZeroTime,
Status: evergreen.TaskUndispatched,
Activated: b.Activated,
RevisionOrderNumber: v.RevisionOrderNumber,
Requester: v.Requester,
Version: v.Id,
Revision: v.Revision,
Project: project.Identifier,
Priority: buildVarTask.Priority,
}
}
// Reset sets the task state to be activated, with a new secret,
// undispatched status and zero time on Start, Scheduled, Dispatch and FinishTime
func ResetTasks(taskIds []string) error {
reset := bson.M{
"$set": bson.M{
ActivatedKey: true,
SecretKey: util.RandomString(),
StatusKey: evergreen.TaskUndispatched,
DispatchTimeKey: util.ZeroTime,
StartTimeKey: util.ZeroTime,
ScheduledTimeKey: util.ZeroTime,
FinishTimeKey: util.ZeroTime,
TestResultsKey: []TestResult{},
},
"$unset": bson.M{
DetailsKey: "",
},
}
_, err := UpdateAll(
bson.M{
IdKey: bson.M{"$in": taskIds},
},
reset,
)
return err
}
func (t *Task) reset() error {
if err := t.Archive(); err != nil {
return fmt.Errorf("Can't restart task because it can't be archived: %v", err)
}
reset := bson.M{
"$set": bson.M{
TaskActivatedKey: true,
TaskSecretKey: util.RandomString(),
TaskStatusKey: evergreen.TaskUndispatched,
TaskDispatchTimeKey: ZeroTime,
TaskStartTimeKey: ZeroTime,
TaskScheduledTimeKey: ZeroTime,
TaskFinishTimeKey: ZeroTime,
TaskTestResultsKey: []TestResult{},
},
"$unset": bson.M{
TaskDetailsKey: "",
},
}
err := UpdateOneTask(bson.M{TaskIdKey: t.Id}, reset)
if err != nil {
return err
}
// update the cached version of the task, in its build document
if err = build.ResetCachedTask(t.BuildId, t.Id); err != nil {
return err
}
return t.UpdateBuildStatus()
}
// Reset sets the task state to be activated, with a new secret,
// undispatched status and zero time on Start, Scheduled, Dispatch and FinishTime
func (t *Task) Reset() error {
t.Activated = true
t.Secret = util.RandomString()
t.DispatchTime = util.ZeroTime
t.StartTime = util.ZeroTime
t.ScheduledTime = util.ZeroTime
t.FinishTime = util.ZeroTime
t.TestResults = []TestResult{}
reset := bson.M{
"$set": bson.M{
ActivatedKey: true,
SecretKey: t.Secret,
StatusKey: evergreen.TaskUndispatched,
DispatchTimeKey: util.ZeroTime,
StartTimeKey: util.ZeroTime,
ScheduledTimeKey: util.ZeroTime,
FinishTimeKey: util.ZeroTime,
TestResultsKey: []TestResult{},
},
"$unset": bson.M{
DetailsKey: "",
},
}
return UpdateOne(
bson.M{
IdKey: t.Id,
},
reset,
)
}
func (staticMgr *MockCloudManager) SpawnInstance(distro *distro.Distro, owner string, userHost bool) (*host.Host, error) {
return &host.Host{
Id: util.RandomString(),
Distro: *distro,
StartedBy: owner,
}, nil
}
func (staticMgr *MockCloudManager) SpawnInstance(distro *distro.Distro, hostOpts cloud.HostOptions) (*host.Host, error) {
return &host.Host{
Id: util.RandomString(),
Distro: *distro,
StartedBy: hostOpts.UserName,
}, nil
}
// NewGithubUserManager initializes a GithubUserManager with a Salt as randomly generated string used in Github
// authentication
func NewGithubUserManager(g *evergreen.GithubAuthConfig) (*GithubUserManager, error) {
if g.ClientId == "" {
return nil, fmt.Errorf("no client id for config")
}
if g.ClientSecret == "" {
return nil, fmt.Errorf("no client secret for config given")
}
return &GithubUserManager{g.ClientId, g.ClientSecret, g.Users, g.Organization, util.RandomString()}, nil
}
func (uis *UIServer) newAPIKey(w http.ResponseWriter, r *http.Request) {
currentUser := MustHaveUser(r)
newKey := util.RandomString()
if err := model.SetUserAPIKey(currentUser.Id, newKey); err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed saving key: %v", err))
return
}
uis.WriteJSON(w, http.StatusOK, struct {
Key string `json:"key"`
}{newKey})
}
func TestPutS3File(t *testing.T) {
testutil.ConfigureIntegrationTest(t, testConfig, "TestPutS3File")
Convey("When given a file to copy to S3...", t, func() {
Convey("a valid source file with a long key should return an error ", func() {
//Make a test file with some random content.
tempfile, err := ioutil.TempFile("", "randomString")
So(err, ShouldEqual, nil)
randStr := util.RandomString()
_, err = tempfile.Write([]byte(randStr))
So(err, ShouldEqual, nil)
tempfile.Close()
// put the test file on S3
auth := &aws.Auth{
AccessKey: testConfig.Providers.AWS.Id,
SecretKey: testConfig.Providers.AWS.Secret,
}
longURLKey := sourceURL + strings.Repeat("suffix", 300)
err = PutS3File(auth, tempfile.Name(), longURLKey, "application/x-tar", "public-read")
So(err, ShouldNotEqual, nil)
})
Convey("a valid source file with a valid key should return no errors", func() {
//Make a test file with some random content.
tempfile, err := ioutil.TempFile("", "randomString")
So(err, ShouldEqual, nil)
randStr := util.RandomString()
_, err = tempfile.Write([]byte(randStr))
So(err, ShouldEqual, nil)
tempfile.Close()
// put the test file on S3
auth := &aws.Auth{
AccessKey: testConfig.Providers.AWS.Id,
SecretKey: testConfig.Providers.AWS.Secret,
}
err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read")
So(err, ShouldEqual, nil)
})
})
}
// GitApplyNumstat attempts to apply a given patch; it returns the patch's bytes
// if it is successful
func GitApplyNumstat(patch string) (*bytes.Buffer, error) {
handle, err := ioutil.TempFile("", util.RandomString())
if err != nil {
return nil, fmt.Errorf("Unable to create local patch file")
}
// convert the patch to bytes
buf := []byte(patch)
buffer := bytes.NewBuffer(buf)
for {
// read a chunk
n, err := buffer.Read(buf)
if err != nil && err != io.EOF {
return nil, fmt.Errorf("Unable to read supplied patch file")
}
if n == 0 {
break
}
// write a chunk
if _, err := handle.Write(buf[:n]); err != nil {
return nil, fmt.Errorf("Unable to read supplied patch file")
}
}
// pseudo-validate the patch set by attempting to get a summary
var summaryBuffer bytes.Buffer
cmd := exec.Command("git", "apply", "--numstat", handle.Name())
cmd.Stdout = &summaryBuffer
cmd.Stderr = &summaryBuffer
cmd.Dir = filepath.Dir(handle.Name())
// this should never happen if patch is initially validated
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("Error validating patch: 424 - %v (%v)",
summaryBuffer.String(), err)
}
// this should never happen if patch is initially validated
if err := cmd.Wait(); err != nil {
return nil, fmt.Errorf("Error waiting on patch: 562 - %v (%v)",
summaryBuffer.String(), err)
}
return &summaryBuffer, nil
}
// GetOrCreateUser fetches a user with the given userId and returns it. If no document exists for
// that userId, inserts it along with the provided display name and email.
func GetOrCreateUser(userId, displayName, email string) (*user.DBUser, error) {
u := &user.DBUser{}
_, err := db.FindAndModify(user.Collection, bson.M{user.IdKey: userId}, nil,
mgo.Change{
Update: bson.M{
"$set": bson.M{
user.DispNameKey: displayName,
user.EmailAddressKey: email,
},
"$setOnInsert": bson.M{
user.APIKeyKey: util.RandomString(),
},
},
ReturnNew: true,
Upsert: true,
}, u)
if err != nil {
return nil, err
}
return u, nil
}
// RestartVersion restarts completed tasks associated with a given versionId.
// If abortInProgress is true, it also sets the abort flag on any in-progress tasks.
func RestartVersion(versionId string, taskIds []string, abortInProgress bool, caller string) error {
// restart all the 'not in-progress' tasks for the version
allTasks, err := FindAllTasks(
bson.M{
TaskIdKey: bson.M{"$in": taskIds},
TaskVersionKey: versionId,
TaskDispatchTimeKey: bson.M{"$ne": ZeroTime},
TaskStatusKey: bson.M{
"$in": []string{
evergreen.TaskSucceeded,
evergreen.TaskFailed,
},
},
},
db.NoProjection,
db.NoSort,
db.NoSkip,
db.NoLimit,
)
if err != nil && err != mgo.ErrNotFound {
return err
}
// archive all the tasks
for _, t := range allTasks {
if err := t.Archive(); err != nil {
return fmt.Errorf("failed to archive task: %v", err)
}
}
// Set all the task fields to indicate restarted
_, err = UpdateAllTasks(
bson.M{TaskIdKey: bson.M{"$in": taskIds}},
bson.M{
"$set": bson.M{
TaskActivatedKey: true,
TaskSecretKey: util.RandomString(),
TaskStatusKey: evergreen.TaskUndispatched,
TaskDispatchTimeKey: ZeroTime,
TaskStartTimeKey: ZeroTime,
TaskScheduledTimeKey: ZeroTime,
TaskFinishTimeKey: ZeroTime,
TaskTestResultsKey: []TestResult{},
},
"$unset": bson.M{
TaskDetailsKey: "",
},
})
if err != nil {
return err
}
// TODO figure out a way to coalesce updates for task cache for the same build, so we
// only need to do one update per-build instead of one per-task here.
// Doesn't seem to be possible as-is because $ can only apply to one array element matched per
// document.
buildIdSet := map[string]bool{}
for _, t := range allTasks {
buildIdSet[t.BuildId] = true
err = build.ResetCachedTask(t.BuildId, t.Id)
if err != nil {
return err
}
}
// reset the build statuses, once per build
buildIdList := make([]string, 0, len(buildIdSet))
for k, _ := range buildIdSet {
buildIdList = append(buildIdList, k)
}
// Set the build status for all the builds containing the tasks that we touched
_, err = build.UpdateAllBuilds(
bson.M{build.IdKey: bson.M{"$in": buildIdList}},
bson.M{"$set": bson.M{build.StatusKey: evergreen.BuildStarted}},
)
if abortInProgress {
// abort in-progress tasks in this build
_, err = UpdateAllTasks(
bson.M{
TaskVersionKey: versionId,
TaskIdKey: bson.M{"$in": taskIds},
TaskStatusKey: bson.M{"$in": evergreen.AbortableStatuses},
},
bson.M{"$set": bson.M{TaskAbortedKey: true}},
)
if err != nil {
return err
}
}
// update activation for all the builds
for _, b := range buildIdList {
err := build.UpdateActivation(b, true, caller)
if err != nil {
return err
}
}
return nil
}
func TestS3Copy(t *testing.T) {
testutil.ConfigureIntegrationTest(t, testConfig, "TestS3Copy")
Convey("When given a source and destination URL to copy from/to...", t, func() {
Convey("a valid source file should be copied to the valid destination", func() {
//Make a test file with some random content.
tempfile, err := ioutil.TempFile("", "randomString")
So(err, ShouldEqual, nil)
randStr := util.RandomString()
_, err = tempfile.Write([]byte(randStr))
So(err, ShouldEqual, nil)
tempfile.Close()
// Put the test file on S3
auth := &aws.Auth{
AccessKey: testConfig.Providers.AWS.Id,
SecretKey: testConfig.Providers.AWS.Secret,
}
err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read")
So(err, ShouldEqual, nil)
// Copy the test file over to another location
err = CopyS3File(auth, sourceURL, destUrl, "public-read")
So(err, ShouldEqual, nil)
// Ensure that the file was actually copied
rdr, err := GetS3File(auth, destUrl)
defer rdr.Close()
So(err, ShouldEqual, nil)
fileContents, err := ioutil.ReadAll(rdr)
So(err, ShouldEqual, nil)
So(string(fileContents), ShouldEqual, randStr)
})
Convey("a valid source file with a long key should return an error "+
"even if sent to a valid destination", func() {
//Make a test file with some random content.
tempfile, err := ioutil.TempFile("", "randomString")
So(err, ShouldEqual, nil)
randStr := util.RandomString()
_, err = tempfile.Write([]byte(randStr))
So(err, ShouldEqual, nil)
tempfile.Close()
// put the test file on S3
auth := &aws.Auth{
AccessKey: testConfig.Providers.AWS.Id,
SecretKey: testConfig.Providers.AWS.Secret,
}
longURLKey := sourceURL + strings.Repeat("suffix", 300)
err = PutS3File(auth, tempfile.Name(), longURLKey, "application/x-tar", "public-read")
So(err, ShouldNotEqual, nil)
})
Convey("a valid source file copied to a destination with too long a "+
"key name should return an error", func() {
//Make a test file with some random content.
tempfile, err := ioutil.TempFile("", "randomString")
So(err, ShouldEqual, nil)
randStr := util.RandomString()
_, err = tempfile.Write([]byte(randStr))
So(err, ShouldEqual, nil)
tempfile.Close()
// put the test file on S3
auth := &aws.Auth{
AccessKey: testConfig.Providers.AWS.Id,
SecretKey: testConfig.Providers.AWS.Secret,
}
err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read")
So(err, ShouldEqual, nil)
longURLKey := sourceURL + strings.Repeat("suffix", 300)
// Attempt to copy the test file over to another location
err = CopyS3File(auth, sourceURL, longURLKey, "public-read")
So(err, ShouldNotEqual, nil)
// Ensure that the file was not copied
rdr, err := GetS3File(auth, longURLKey)
So(rdr, ShouldEqual, nil)
So(err, ShouldNotEqual, nil)
})
})
}
func TestS3PutAndGet(t *testing.T) {
testutil.HandleTestingErr(
db.ClearCollections(task.Collection, artifact.Collection), t,
"error clearing test collections")
conf := evergreen.TestConfig()
testutil.ConfigureIntegrationTest(t, conf, "TestS3PutAndGet")
Convey("When putting to and retrieving from an s3 bucket", t, func() {
var putCmd *S3PutCommand
var getCmd *S3GetCommand
testDataDir := "testdata"
remoteFile := "remote_mci_put_test.tgz"
bucket := "mci-test-uploads"
permissions := "private"
contentType := "application/x-tar"
displayName := "testfile"
// create the local directory to be tarred
localDirToTar := filepath.Join(testDataDir, "put_test")
localFileToTar := filepath.Join(localDirToTar, "put_test_file.txt")
testutil.HandleTestingErr(os.RemoveAll(localDirToTar), t, "Error removing"+
" directory")
testutil.HandleTestingErr(os.MkdirAll(localDirToTar, 0755), t,
"Error creating directory")
randStr := util.RandomString()
So(ioutil.WriteFile(localFileToTar, []byte(randStr), 0755), ShouldBeNil)
// tar it
tarCmd := &command.LocalCommand{
CmdString: "tar czf put_test.tgz put_test",
WorkingDirectory: testDataDir,
Stdout: ioutil.Discard,
Stderr: ioutil.Discard,
}
testutil.HandleTestingErr(tarCmd.Run(), t, "Error tarring directories")
tarballSource := filepath.Join(testDataDir, "put_test.tgz")
// remove the untarred version
testutil.HandleTestingErr(os.RemoveAll(localDirToTar), t, "Error removing directories")
Convey("the file retrieved should be the exact same as the file put", func() {
// load params into the put command
putCmd = &S3PutCommand{}
putParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"local_file": tarballSource,
"remote_file": remoteFile,
"bucket": bucket,
"permissions": permissions,
"content_type": contentType,
"display_name": displayName,
}
So(putCmd.ParseParams(putParams), ShouldBeNil)
So(putCmd.Put(), ShouldBeNil)
// next, get the file, untarring it
getCmd = &S3GetCommand{}
getParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"remote_file": remoteFile,
"bucket": bucket,
"extract_to": testDataDir,
"display_name": displayName,
}
So(getCmd.ParseParams(getParams), ShouldBeNil)
So(getCmd.Get(), ShouldBeNil)
// read in the file that was pulled down
fileContents, err := ioutil.ReadFile(localFileToTar)
So(err, ShouldBeNil)
So(string(fileContents), ShouldEqual, randStr)
// now, get the tarball without untarring it
getCmd = &S3GetCommand{}
localDlTarget := filepath.Join(testDataDir, "put_test_dl.tgz")
getParams = map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"remote_file": remoteFile,
"bucket": bucket,
"local_file": localDlTarget,
"display_name": displayName,
}
So(getCmd.ParseParams(getParams), ShouldBeNil)
So(getCmd.Get(), ShouldBeNil)
exists, err := util.FileExists(localDlTarget)
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
})
Convey("the put command should always run if there is no variants filter", func() {
// load params into the put command
putCmd = &S3PutCommand{}
putParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"local_file": tarballSource,
"remote_file": remoteFile,
"bucket": bucket,
"permissions": permissions,
"content_type": contentType,
"display_name": displayName,
}
So(putCmd.ParseParams(putParams), ShouldBeNil)
So(putCmd.shouldRunForVariant("linux-64"), ShouldBeTrue)
})
Convey("put cmd with variants filter should only run if variant is in list", func() {
// load params into the put command
putCmd = &S3PutCommand{}
putParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"local_file": tarballSource,
"remote_file": remoteFile,
"bucket": bucket,
"permissions": permissions,
"content_type": contentType,
"display_name": displayName,
"build_variants": []string{"linux-64", "windows-64"},
}
So(putCmd.ParseParams(putParams), ShouldBeNil)
So(putCmd.shouldRunForVariant("linux-64"), ShouldBeTrue)
So(putCmd.shouldRunForVariant("osx-108"), ShouldBeFalse)
})
Convey("put cmd with 'optional' and missing file should not throw an error", func() {
// load params into the put command
putCmd = &S3PutCommand{}
putParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"optional": true,
"local_file": "this_file_does_not_exist.txt",
"remote_file": "remote_file",
"bucket": "test_bucket",
"permissions": "private",
"content_type": "text/plain",
}
So(putCmd.ParseParams(putParams), ShouldBeNil)
server, err := apiserver.CreateTestServer(conf, nil, plugin.APIPlugins, false)
httpCom := plugintest.TestAgentCommunicator("testTask", "taskSecret", server.URL)
pluginCom := &agent.TaskJSONCommunicator{"s3", httpCom}
So(err, ShouldBeNil)
err = putCmd.Execute(&plugintest.MockLogger{}, pluginCom,
&model.TaskConfig{nil, nil, nil, nil, &model.BuildVariant{Name: "linux"}, &command.Expansions{}, "."}, make(chan bool))
So(err, ShouldBeNil)
})
})
}
func TestS3PutAndGetMultiFile(t *testing.T) {
testutil.HandleTestingErr(
db.ClearCollections(task.Collection, artifact.Collection), t,
"error clearing test collections")
conf := evergreen.TestConfig()
testutil.ConfigureIntegrationTest(t, conf, "TestS3PutAndGet")
Convey("When putting to and retrieving from an s3 bucket", t, func() {
testDataDir := "testdata"
bucket := "mci-test-uploads"
permissions := "private"
contentType := "application/x-tar"
remoteFilePrefix := "remote_mci_put_test-"
displayNamePrefix := "testfile-"
// create the local directory
localDir := filepath.Join(testDataDir, "put_test")
localFilePrefix := filepath.Join(localDir, "put_test_file")
defer func() {
testutil.HandleTestingErr(os.RemoveAll(testDataDir), t, "error removing test dir")
}()
testutil.HandleTestingErr(os.RemoveAll(localDir), t,
"Error removing directory")
testutil.HandleTestingErr(os.MkdirAll(localDir, 0755), t,
"Error creating directory")
fileInfos := []multiPutFileInfo{}
for i := 0; i < 5; i++ {
randStr := util.RandomString()
localPath := fmt.Sprintf("%s%d.txt", localFilePrefix, i)
localName := filepath.Base(localPath)
remoteName := fmt.Sprintf("%s%s", remoteFilePrefix, localName)
displayName := fmt.Sprintf("%s%s", displayNamePrefix, localName)
So(ioutil.WriteFile(localPath, []byte(randStr), 0755), ShouldBeNil)
fileInfo := multiPutFileInfo{
remoteName: remoteName,
localPath: localPath,
displayName: displayName,
data: randStr,
}
fileInfos = append(fileInfos, fileInfo)
}
Convey("the files should be put without error", func() {
includesFilter := []string{fmt.Sprintf("%s/*.txt", localDir)}
// load params into the put command
putCmd := &S3PutCommand{}
putParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"local_files_include_filter": includesFilter,
"remote_file": remoteFilePrefix,
"bucket": bucket,
"permissions": permissions,
"content_type": contentType,
"display_name": displayNamePrefix,
}
So(putCmd.ParseParams(putParams), ShouldBeNil)
So(putCmd.Put(), ShouldBeNil)
Convey("the files should each be gotten without error", func() {
for _, f := range fileInfos {
// next, get the file
getCmd := &S3GetCommand{}
getParams := map[string]interface{}{
"aws_key": conf.Providers.AWS.Id,
"aws_secret": conf.Providers.AWS.Secret,
"remote_file": f.remoteName,
"bucket": bucket,
"local_file": f.localPath,
"display_name": f.displayName,
}
So(getCmd.ParseParams(getParams), ShouldBeNil)
So(getCmd.Get(), ShouldBeNil)
// read in the file that was pulled down
fileContents, err := ioutil.ReadFile(f.localPath)
So(err, ShouldBeNil)
So(string(fileContents), ShouldEqual, f.data)
}
})
})
})
}