Ejemplo n.º 1
0
// createOneTask is a helper to create a single task.
func createOneTask(id string, buildVarTask BuildVariantTask, project *Project,
	buildVariant *BuildVariant, b *build.Build, v *version.Version) *Task {
	return &Task{
		Id:                  id,
		Secret:              util.RandomString(),
		DisplayName:         buildVarTask.Name,
		BuildId:             b.Id,
		BuildVariant:        buildVariant.Name,
		CreateTime:          b.CreateTime,
		PushTime:            b.PushTime,
		ScheduledTime:       ZeroTime,
		StartTime:           ZeroTime, // Certain time fields must be initialized
		FinishTime:          ZeroTime, // to our own ZeroTime value (which is
		DispatchTime:        ZeroTime, // Unix epoch 0, not Go's time.Time{})
		LastHeartbeat:       ZeroTime,
		Status:              evergreen.TaskUndispatched,
		Activated:           b.Activated,
		RevisionOrderNumber: v.RevisionOrderNumber,
		Requester:           v.Requester,
		Version:             v.Id,
		Revision:            v.Revision,
		Project:             project.Identifier,
		Priority:            buildVarTask.Priority,
	}
}
Ejemplo n.º 2
0
// Reset sets the task state to be activated, with a new secret,
// undispatched status and zero time on Start, Scheduled, Dispatch and FinishTime
func ResetTasks(taskIds []string) error {
	reset := bson.M{
		"$set": bson.M{
			ActivatedKey:     true,
			SecretKey:        util.RandomString(),
			StatusKey:        evergreen.TaskUndispatched,
			DispatchTimeKey:  util.ZeroTime,
			StartTimeKey:     util.ZeroTime,
			ScheduledTimeKey: util.ZeroTime,
			FinishTimeKey:    util.ZeroTime,
			TestResultsKey:   []TestResult{},
		},
		"$unset": bson.M{
			DetailsKey: "",
		},
	}

	_, err := UpdateAll(
		bson.M{
			IdKey: bson.M{"$in": taskIds},
		},
		reset,
	)
	return err

}
Ejemplo n.º 3
0
func (t *Task) reset() error {
	if err := t.Archive(); err != nil {
		return fmt.Errorf("Can't restart task because it can't be archived: %v", err)
	}

	reset := bson.M{
		"$set": bson.M{
			TaskActivatedKey:     true,
			TaskSecretKey:        util.RandomString(),
			TaskStatusKey:        evergreen.TaskUndispatched,
			TaskDispatchTimeKey:  ZeroTime,
			TaskStartTimeKey:     ZeroTime,
			TaskScheduledTimeKey: ZeroTime,
			TaskFinishTimeKey:    ZeroTime,
			TaskTestResultsKey:   []TestResult{},
		},
		"$unset": bson.M{
			TaskDetailsKey: "",
		},
	}

	err := UpdateOneTask(bson.M{TaskIdKey: t.Id}, reset)
	if err != nil {
		return err
	}

	// update the cached version of the task, in its build document
	if err = build.ResetCachedTask(t.BuildId, t.Id); err != nil {
		return err
	}

	return t.UpdateBuildStatus()
}
Ejemplo n.º 4
0
// Reset sets the task state to be activated, with a new secret,
// undispatched status and zero time on Start, Scheduled, Dispatch and FinishTime
func (t *Task) Reset() error {
	t.Activated = true
	t.Secret = util.RandomString()
	t.DispatchTime = util.ZeroTime
	t.StartTime = util.ZeroTime
	t.ScheduledTime = util.ZeroTime
	t.FinishTime = util.ZeroTime
	t.TestResults = []TestResult{}
	reset := bson.M{
		"$set": bson.M{
			ActivatedKey:     true,
			SecretKey:        t.Secret,
			StatusKey:        evergreen.TaskUndispatched,
			DispatchTimeKey:  util.ZeroTime,
			StartTimeKey:     util.ZeroTime,
			ScheduledTimeKey: util.ZeroTime,
			FinishTimeKey:    util.ZeroTime,
			TestResultsKey:   []TestResult{},
		},
		"$unset": bson.M{
			DetailsKey: "",
		},
	}

	return UpdateOne(
		bson.M{
			IdKey: t.Id,
		},
		reset,
	)
}
Ejemplo n.º 5
0
func (staticMgr *MockCloudManager) SpawnInstance(distro *distro.Distro, owner string, userHost bool) (*host.Host, error) {
	return &host.Host{
		Id:        util.RandomString(),
		Distro:    *distro,
		StartedBy: owner,
	}, nil
}
Ejemplo n.º 6
0
func (staticMgr *MockCloudManager) SpawnInstance(distro *distro.Distro, hostOpts cloud.HostOptions) (*host.Host, error) {
	return &host.Host{
		Id:        util.RandomString(),
		Distro:    *distro,
		StartedBy: hostOpts.UserName,
	}, nil
}
Ejemplo n.º 7
0
// NewGithubUserManager initializes a GithubUserManager with a Salt as randomly generated string used in Github
// authentication
func NewGithubUserManager(g *evergreen.GithubAuthConfig) (*GithubUserManager, error) {
	if g.ClientId == "" {
		return nil, fmt.Errorf("no client id for config")
	}
	if g.ClientSecret == "" {
		return nil, fmt.Errorf("no client secret for config given")
	}
	return &GithubUserManager{g.ClientId, g.ClientSecret, g.Users, g.Organization, util.RandomString()}, nil
}
Ejemplo n.º 8
0
func (uis *UIServer) newAPIKey(w http.ResponseWriter, r *http.Request) {
	currentUser := MustHaveUser(r)
	newKey := util.RandomString()
	if err := model.SetUserAPIKey(currentUser.Id, newKey); err != nil {
		uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed saving key: %v", err))
		return
	}
	uis.WriteJSON(w, http.StatusOK, struct {
		Key string `json:"key"`
	}{newKey})
}
Ejemplo n.º 9
0
func TestPutS3File(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, testConfig, "TestPutS3File")
	Convey("When given a file to copy to S3...", t, func() {
		Convey("a valid source file with a long key should return an error ", func() {
			//Make a test file with some random content.
			tempfile, err := ioutil.TempFile("", "randomString")
			So(err, ShouldEqual, nil)
			randStr := util.RandomString()
			_, err = tempfile.Write([]byte(randStr))
			So(err, ShouldEqual, nil)
			tempfile.Close()

			// put the test file on S3
			auth := &aws.Auth{
				AccessKey: testConfig.Providers.AWS.Id,
				SecretKey: testConfig.Providers.AWS.Secret,
			}
			longURLKey := sourceURL + strings.Repeat("suffix", 300)
			err = PutS3File(auth, tempfile.Name(), longURLKey, "application/x-tar", "public-read")
			So(err, ShouldNotEqual, nil)
		})
		Convey("a valid source file with a valid key should return no errors", func() {
			//Make a test file with some random content.
			tempfile, err := ioutil.TempFile("", "randomString")
			So(err, ShouldEqual, nil)
			randStr := util.RandomString()
			_, err = tempfile.Write([]byte(randStr))
			So(err, ShouldEqual, nil)
			tempfile.Close()

			// put the test file on S3
			auth := &aws.Auth{
				AccessKey: testConfig.Providers.AWS.Id,
				SecretKey: testConfig.Providers.AWS.Secret,
			}
			err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read")
			So(err, ShouldEqual, nil)
		})
	})
}
Ejemplo n.º 10
0
// GitApplyNumstat attempts to apply a given patch; it returns the patch's bytes
// if it is successful
func GitApplyNumstat(patch string) (*bytes.Buffer, error) {
	handle, err := ioutil.TempFile("", util.RandomString())
	if err != nil {
		return nil, fmt.Errorf("Unable to create local patch file")
	}
	// convert the patch to bytes
	buf := []byte(patch)
	buffer := bytes.NewBuffer(buf)
	for {
		// read a chunk
		n, err := buffer.Read(buf)
		if err != nil && err != io.EOF {
			return nil, fmt.Errorf("Unable to read supplied patch file")
		}
		if n == 0 {
			break
		}
		// write a chunk
		if _, err := handle.Write(buf[:n]); err != nil {
			return nil, fmt.Errorf("Unable to read supplied patch file")
		}
	}

	// pseudo-validate the patch set by attempting to get a summary
	var summaryBuffer bytes.Buffer
	cmd := exec.Command("git", "apply", "--numstat", handle.Name())
	cmd.Stdout = &summaryBuffer
	cmd.Stderr = &summaryBuffer
	cmd.Dir = filepath.Dir(handle.Name())

	// this should never happen if patch is initially validated
	if err := cmd.Start(); err != nil {
		return nil, fmt.Errorf("Error validating patch: 424 - %v (%v)",
			summaryBuffer.String(), err)
	}

	// this should never happen if patch is initially validated
	if err := cmd.Wait(); err != nil {
		return nil, fmt.Errorf("Error waiting on patch: 562 - %v (%v)",
			summaryBuffer.String(), err)
	}
	return &summaryBuffer, nil
}
Ejemplo n.º 11
0
// GetOrCreateUser fetches a user with the given userId and returns it. If no document exists for
// that userId, inserts it along with the provided display name and email.
func GetOrCreateUser(userId, displayName, email string) (*user.DBUser, error) {
	u := &user.DBUser{}
	_, err := db.FindAndModify(user.Collection, bson.M{user.IdKey: userId}, nil,
		mgo.Change{
			Update: bson.M{
				"$set": bson.M{
					user.DispNameKey:     displayName,
					user.EmailAddressKey: email,
				},
				"$setOnInsert": bson.M{
					user.APIKeyKey: util.RandomString(),
				},
			},
			ReturnNew: true,
			Upsert:    true,
		}, u)
	if err != nil {
		return nil, err
	}
	return u, nil
}
Ejemplo n.º 12
0
// RestartVersion restarts completed tasks associated with a given versionId.
// If abortInProgress is true, it also sets the abort flag on any in-progress tasks.
func RestartVersion(versionId string, taskIds []string, abortInProgress bool, caller string) error {
	// restart all the 'not in-progress' tasks for the version
	allTasks, err := FindAllTasks(
		bson.M{
			TaskIdKey:           bson.M{"$in": taskIds},
			TaskVersionKey:      versionId,
			TaskDispatchTimeKey: bson.M{"$ne": ZeroTime},
			TaskStatusKey: bson.M{
				"$in": []string{
					evergreen.TaskSucceeded,
					evergreen.TaskFailed,
				},
			},
		},
		db.NoProjection,
		db.NoSort,
		db.NoSkip,
		db.NoLimit,
	)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}

	// archive all the tasks
	for _, t := range allTasks {
		if err := t.Archive(); err != nil {
			return fmt.Errorf("failed to archive task: %v", err)
		}
	}

	// Set all the task fields to indicate restarted
	_, err = UpdateAllTasks(
		bson.M{TaskIdKey: bson.M{"$in": taskIds}},
		bson.M{
			"$set": bson.M{
				TaskActivatedKey:     true,
				TaskSecretKey:        util.RandomString(),
				TaskStatusKey:        evergreen.TaskUndispatched,
				TaskDispatchTimeKey:  ZeroTime,
				TaskStartTimeKey:     ZeroTime,
				TaskScheduledTimeKey: ZeroTime,
				TaskFinishTimeKey:    ZeroTime,
				TaskTestResultsKey:   []TestResult{},
			},
			"$unset": bson.M{
				TaskDetailsKey: "",
			},
		})
	if err != nil {
		return err
	}

	// TODO figure out a way to coalesce updates for task cache for the same build, so we
	// only need to do one update per-build instead of one per-task here.
	// Doesn't seem to be possible as-is because $ can only apply to one array element matched per
	// document.
	buildIdSet := map[string]bool{}
	for _, t := range allTasks {
		buildIdSet[t.BuildId] = true
		err = build.ResetCachedTask(t.BuildId, t.Id)
		if err != nil {
			return err
		}
	}

	// reset the build statuses, once per build
	buildIdList := make([]string, 0, len(buildIdSet))
	for k, _ := range buildIdSet {
		buildIdList = append(buildIdList, k)
	}

	// Set the build status for all the builds containing the tasks that we touched
	_, err = build.UpdateAllBuilds(
		bson.M{build.IdKey: bson.M{"$in": buildIdList}},
		bson.M{"$set": bson.M{build.StatusKey: evergreen.BuildStarted}},
	)

	if abortInProgress {
		// abort in-progress tasks in this build
		_, err = UpdateAllTasks(
			bson.M{
				TaskVersionKey: versionId,
				TaskIdKey:      bson.M{"$in": taskIds},
				TaskStatusKey:  bson.M{"$in": evergreen.AbortableStatuses},
			},
			bson.M{"$set": bson.M{TaskAbortedKey: true}},
		)
		if err != nil {
			return err
		}
	}

	// update activation for all the builds
	for _, b := range buildIdList {
		err := build.UpdateActivation(b, true, caller)
		if err != nil {
			return err
		}
	}
	return nil

}
Ejemplo n.º 13
0
func TestS3Copy(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, testConfig, "TestS3Copy")
	Convey("When given a source and  destination URL to copy from/to...", t, func() {
		Convey("a valid source file should be copied to the valid destination", func() {
			//Make a test file with some random content.
			tempfile, err := ioutil.TempFile("", "randomString")
			So(err, ShouldEqual, nil)
			randStr := util.RandomString()

			_, err = tempfile.Write([]byte(randStr))
			So(err, ShouldEqual, nil)
			tempfile.Close()

			// Put the test file on S3
			auth := &aws.Auth{
				AccessKey: testConfig.Providers.AWS.Id,
				SecretKey: testConfig.Providers.AWS.Secret,
			}
			err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read")
			So(err, ShouldEqual, nil)

			// Copy the test file over to another location
			err = CopyS3File(auth, sourceURL, destUrl, "public-read")
			So(err, ShouldEqual, nil)

			// Ensure that the file was actually copied
			rdr, err := GetS3File(auth, destUrl)
			defer rdr.Close()
			So(err, ShouldEqual, nil)

			fileContents, err := ioutil.ReadAll(rdr)
			So(err, ShouldEqual, nil)
			So(string(fileContents), ShouldEqual, randStr)
		})
		Convey("a valid source file with a long key should return an error "+
			"even if sent to a valid destination", func() {
			//Make a test file with some random content.
			tempfile, err := ioutil.TempFile("", "randomString")
			So(err, ShouldEqual, nil)
			randStr := util.RandomString()
			_, err = tempfile.Write([]byte(randStr))
			So(err, ShouldEqual, nil)
			tempfile.Close()

			// put the test file on S3
			auth := &aws.Auth{
				AccessKey: testConfig.Providers.AWS.Id,
				SecretKey: testConfig.Providers.AWS.Secret,
			}
			longURLKey := sourceURL + strings.Repeat("suffix", 300)
			err = PutS3File(auth, tempfile.Name(), longURLKey, "application/x-tar", "public-read")
			So(err, ShouldNotEqual, nil)
		})
		Convey("a valid source file copied to a destination with too long a "+
			"key name should return an error", func() {
			//Make a test file with some random content.
			tempfile, err := ioutil.TempFile("", "randomString")
			So(err, ShouldEqual, nil)
			randStr := util.RandomString()
			_, err = tempfile.Write([]byte(randStr))
			So(err, ShouldEqual, nil)
			tempfile.Close()

			// put the test file on S3
			auth := &aws.Auth{
				AccessKey: testConfig.Providers.AWS.Id,
				SecretKey: testConfig.Providers.AWS.Secret,
			}
			err = PutS3File(auth, tempfile.Name(), sourceURL, "application/x-tar", "public-read")
			So(err, ShouldEqual, nil)

			longURLKey := sourceURL + strings.Repeat("suffix", 300)
			// Attempt to copy the test file over to another location
			err = CopyS3File(auth, sourceURL, longURLKey, "public-read")
			So(err, ShouldNotEqual, nil)

			// Ensure that the file was not copied
			rdr, err := GetS3File(auth, longURLKey)
			So(rdr, ShouldEqual, nil)
			So(err, ShouldNotEqual, nil)
		})
	})
}
Ejemplo n.º 14
0
func TestS3PutAndGet(t *testing.T) {
	testutil.HandleTestingErr(
		db.ClearCollections(task.Collection, artifact.Collection), t,
		"error clearing test collections")

	conf := evergreen.TestConfig()
	testutil.ConfigureIntegrationTest(t, conf, "TestS3PutAndGet")

	Convey("When putting to and retrieving from an s3 bucket", t, func() {

		var putCmd *S3PutCommand
		var getCmd *S3GetCommand

		testDataDir := "testdata"
		remoteFile := "remote_mci_put_test.tgz"
		bucket := "mci-test-uploads"
		permissions := "private"
		contentType := "application/x-tar"
		displayName := "testfile"

		// create the local directory to be tarred
		localDirToTar := filepath.Join(testDataDir, "put_test")
		localFileToTar := filepath.Join(localDirToTar, "put_test_file.txt")
		testutil.HandleTestingErr(os.RemoveAll(localDirToTar), t, "Error removing"+
			" directory")
		testutil.HandleTestingErr(os.MkdirAll(localDirToTar, 0755), t,
			"Error creating directory")
		randStr := util.RandomString()
		So(ioutil.WriteFile(localFileToTar, []byte(randStr), 0755), ShouldBeNil)

		// tar it
		tarCmd := &command.LocalCommand{
			CmdString:        "tar czf put_test.tgz put_test",
			WorkingDirectory: testDataDir,
			Stdout:           ioutil.Discard,
			Stderr:           ioutil.Discard,
		}
		testutil.HandleTestingErr(tarCmd.Run(), t, "Error tarring directories")
		tarballSource := filepath.Join(testDataDir, "put_test.tgz")

		// remove the untarred version
		testutil.HandleTestingErr(os.RemoveAll(localDirToTar), t, "Error removing directories")

		Convey("the file retrieved should be the exact same as the file put", func() {

			// load params into the put command
			putCmd = &S3PutCommand{}
			putParams := map[string]interface{}{
				"aws_key":      conf.Providers.AWS.Id,
				"aws_secret":   conf.Providers.AWS.Secret,
				"local_file":   tarballSource,
				"remote_file":  remoteFile,
				"bucket":       bucket,
				"permissions":  permissions,
				"content_type": contentType,
				"display_name": displayName,
			}
			So(putCmd.ParseParams(putParams), ShouldBeNil)
			So(putCmd.Put(), ShouldBeNil)

			// next, get the file, untarring it
			getCmd = &S3GetCommand{}
			getParams := map[string]interface{}{
				"aws_key":      conf.Providers.AWS.Id,
				"aws_secret":   conf.Providers.AWS.Secret,
				"remote_file":  remoteFile,
				"bucket":       bucket,
				"extract_to":   testDataDir,
				"display_name": displayName,
			}
			So(getCmd.ParseParams(getParams), ShouldBeNil)
			So(getCmd.Get(), ShouldBeNil)
			// read in the file that was pulled down
			fileContents, err := ioutil.ReadFile(localFileToTar)
			So(err, ShouldBeNil)
			So(string(fileContents), ShouldEqual, randStr)

			// now, get the tarball without untarring it
			getCmd = &S3GetCommand{}
			localDlTarget := filepath.Join(testDataDir, "put_test_dl.tgz")
			getParams = map[string]interface{}{
				"aws_key":      conf.Providers.AWS.Id,
				"aws_secret":   conf.Providers.AWS.Secret,
				"remote_file":  remoteFile,
				"bucket":       bucket,
				"local_file":   localDlTarget,
				"display_name": displayName,
			}
			So(getCmd.ParseParams(getParams), ShouldBeNil)
			So(getCmd.Get(), ShouldBeNil)
			exists, err := util.FileExists(localDlTarget)
			So(err, ShouldBeNil)
			So(exists, ShouldBeTrue)

		})

		Convey("the put command should always run if there is no variants filter", func() {
			// load params into the put command
			putCmd = &S3PutCommand{}
			putParams := map[string]interface{}{
				"aws_key":      conf.Providers.AWS.Id,
				"aws_secret":   conf.Providers.AWS.Secret,
				"local_file":   tarballSource,
				"remote_file":  remoteFile,
				"bucket":       bucket,
				"permissions":  permissions,
				"content_type": contentType,
				"display_name": displayName,
			}
			So(putCmd.ParseParams(putParams), ShouldBeNil)
			So(putCmd.shouldRunForVariant("linux-64"), ShouldBeTrue)
		})

		Convey("put cmd with variants filter should only run if variant is in list", func() {
			// load params into the put command
			putCmd = &S3PutCommand{}
			putParams := map[string]interface{}{
				"aws_key":        conf.Providers.AWS.Id,
				"aws_secret":     conf.Providers.AWS.Secret,
				"local_file":     tarballSource,
				"remote_file":    remoteFile,
				"bucket":         bucket,
				"permissions":    permissions,
				"content_type":   contentType,
				"display_name":   displayName,
				"build_variants": []string{"linux-64", "windows-64"},
			}
			So(putCmd.ParseParams(putParams), ShouldBeNil)

			So(putCmd.shouldRunForVariant("linux-64"), ShouldBeTrue)
			So(putCmd.shouldRunForVariant("osx-108"), ShouldBeFalse)
		})

		Convey("put cmd with 'optional' and missing file should not throw an error", func() {
			// load params into the put command
			putCmd = &S3PutCommand{}
			putParams := map[string]interface{}{
				"aws_key":      conf.Providers.AWS.Id,
				"aws_secret":   conf.Providers.AWS.Secret,
				"optional":     true,
				"local_file":   "this_file_does_not_exist.txt",
				"remote_file":  "remote_file",
				"bucket":       "test_bucket",
				"permissions":  "private",
				"content_type": "text/plain",
			}
			So(putCmd.ParseParams(putParams), ShouldBeNil)
			server, err := apiserver.CreateTestServer(conf, nil, plugin.APIPlugins, false)
			httpCom := plugintest.TestAgentCommunicator("testTask", "taskSecret", server.URL)
			pluginCom := &agent.TaskJSONCommunicator{"s3", httpCom}

			So(err, ShouldBeNil)

			err = putCmd.Execute(&plugintest.MockLogger{}, pluginCom,
				&model.TaskConfig{nil, nil, nil, nil, &model.BuildVariant{Name: "linux"}, &command.Expansions{}, "."}, make(chan bool))
			So(err, ShouldBeNil)
		})
	})

}
Ejemplo n.º 15
0
func TestS3PutAndGetMultiFile(t *testing.T) {
	testutil.HandleTestingErr(
		db.ClearCollections(task.Collection, artifact.Collection), t,
		"error clearing test collections")

	conf := evergreen.TestConfig()
	testutil.ConfigureIntegrationTest(t, conf, "TestS3PutAndGet")

	Convey("When putting to and retrieving from an s3 bucket", t, func() {

		testDataDir := "testdata"
		bucket := "mci-test-uploads"
		permissions := "private"
		contentType := "application/x-tar"
		remoteFilePrefix := "remote_mci_put_test-"
		displayNamePrefix := "testfile-"

		// create the local directory
		localDir := filepath.Join(testDataDir, "put_test")
		localFilePrefix := filepath.Join(localDir, "put_test_file")
		defer func() {
			testutil.HandleTestingErr(os.RemoveAll(testDataDir), t, "error removing test dir")
		}()
		testutil.HandleTestingErr(os.RemoveAll(localDir), t,
			"Error removing directory")
		testutil.HandleTestingErr(os.MkdirAll(localDir, 0755), t,
			"Error creating directory")
		fileInfos := []multiPutFileInfo{}
		for i := 0; i < 5; i++ {
			randStr := util.RandomString()
			localPath := fmt.Sprintf("%s%d.txt", localFilePrefix, i)
			localName := filepath.Base(localPath)
			remoteName := fmt.Sprintf("%s%s", remoteFilePrefix, localName)
			displayName := fmt.Sprintf("%s%s", displayNamePrefix, localName)
			So(ioutil.WriteFile(localPath, []byte(randStr), 0755), ShouldBeNil)
			fileInfo := multiPutFileInfo{
				remoteName:  remoteName,
				localPath:   localPath,
				displayName: displayName,
				data:        randStr,
			}
			fileInfos = append(fileInfos, fileInfo)
		}

		Convey("the files should be put without error", func() {
			includesFilter := []string{fmt.Sprintf("%s/*.txt", localDir)}

			// load params into the put command
			putCmd := &S3PutCommand{}
			putParams := map[string]interface{}{
				"aws_key":                    conf.Providers.AWS.Id,
				"aws_secret":                 conf.Providers.AWS.Secret,
				"local_files_include_filter": includesFilter,
				"remote_file":                remoteFilePrefix,
				"bucket":                     bucket,
				"permissions":                permissions,
				"content_type":               contentType,
				"display_name":               displayNamePrefix,
			}
			So(putCmd.ParseParams(putParams), ShouldBeNil)
			So(putCmd.Put(), ShouldBeNil)
			Convey("the files should each be gotten without error", func() {
				for _, f := range fileInfos {
					// next, get the file
					getCmd := &S3GetCommand{}
					getParams := map[string]interface{}{
						"aws_key":      conf.Providers.AWS.Id,
						"aws_secret":   conf.Providers.AWS.Secret,
						"remote_file":  f.remoteName,
						"bucket":       bucket,
						"local_file":   f.localPath,
						"display_name": f.displayName,
					}
					So(getCmd.ParseParams(getParams), ShouldBeNil)
					So(getCmd.Get(), ShouldBeNil)

					// read in the file that was pulled down
					fileContents, err := ioutil.ReadFile(f.localPath)
					So(err, ShouldBeNil)
					So(string(fileContents), ShouldEqual, f.data)

				}
			})
		})

	})
}