// Build the setup script that will need to be run on the specified host.
func (init *HostInit) expandScript(s string) (string, error) {
// replace expansions in the script
exp := command.NewExpansions(init.Settings.Expansions)
script, err := exp.ExpandString(s)
if err != nil {
return "", fmt.Errorf("expansions error: %v", err)
}
return script, err
}
// Build the setup script that will need to be run on the specified host.
func (init *HostInit) buildSetupScript(h *host.Host) (string, error) {
// replace expansions in the script
exp := command.NewExpansions(init.Settings.Expansions)
setupScript, err := exp.ExpandString(h.Distro.Setup)
if err != nil {
return "", fmt.Errorf("expansions error: %v", err)
}
return setupScript, err
}
func TestExpandS3PutParams(t *testing.T) {
Convey("With an s3 put command and a task config", t, func() {
var cmd *S3PutCommand
var conf *model.TaskConfig
Convey("when expanding the command's params", func() {
cmd = &S3PutCommand{}
conf = &model.TaskConfig{
Expansions: command.NewExpansions(map[string]string{}),
}
Convey("all appropriate values should be expanded, if they"+
" contain expansions", func() {
cmd.AwsKey = "${aws_key}"
cmd.AwsSecret = "${aws_secret}"
cmd.RemoteFile = "${remote_file}"
cmd.Bucket = "${bucket}"
cmd.ContentType = "${content_type}"
cmd.DisplayName = "${display_name}"
cmd.Visibility = "${visibility}"
conf.Expansions.Update(
map[string]string{
"aws_key": "key",
"aws_secret": "secret",
"remote_file": "remote",
"bucket": "bck",
"content_type": "ct",
"display_name": "file",
"visibility": artifact.Private,
},
)
So(cmd.expandParams(conf), ShouldBeNil)
So(cmd.AwsKey, ShouldEqual, "key")
So(cmd.AwsSecret, ShouldEqual, "secret")
So(cmd.RemoteFile, ShouldEqual, "remote")
So(cmd.Bucket, ShouldEqual, "bck")
So(cmd.ContentType, ShouldEqual, "ct")
So(cmd.DisplayName, ShouldEqual, "file")
So(cmd.Visibility, ShouldEqual, "private")
})
})
})
}
func populateExpansions(d *distro.Distro, bv *BuildVariant, t *Task) *command.Expansions {
expansions := command.NewExpansions(map[string]string{})
expansions.Put("execution", fmt.Sprintf("%v", t.Execution))
expansions.Put("task_id", t.Id)
expansions.Put("task_name", t.DisplayName)
expansions.Put("build_id", t.BuildId)
expansions.Put("build_variant", t.BuildVariant)
expansions.Put("workdir", d.WorkDir)
expansions.Put("revision", t.Revision)
expansions.Put("project", t.Project)
expansions.Put("branch_name", t.Project)
for _, e := range d.Expansions {
expansions.Put(e.Key, e.Value)
}
expansions.Update(bv.Expansions)
return expansions
}
func TestExpandS3GetParams(t *testing.T) {
Convey("With an s3 get command and a task config", t, func() {
var cmd *S3GetCommand
var conf *model.TaskConfig
Convey("when expanding the command's params", func() {
cmd = &S3GetCommand{}
conf = &model.TaskConfig{
Expansions: command.NewExpansions(map[string]string{}),
}
Convey("all appropriate values should be expanded, if they"+
" contain expansions", func() {
cmd.AwsKey = "${aws_key}"
cmd.AwsSecret = "${aws_secret}"
cmd.RemoteFile = "${remote_file}"
cmd.Bucket = "${bucket}"
conf.Expansions.Update(
map[string]string{
"aws_key": "key",
"aws_secret": "secret",
"remote_file": "remote",
"bucket": "bck",
},
)
So(cmd.expandParams(conf), ShouldBeNil)
So(cmd.AwsKey, ShouldEqual, "key")
So(cmd.AwsSecret, ShouldEqual, "secret")
So(cmd.RemoteFile, ShouldEqual, "remote")
So(cmd.Bucket, ShouldEqual, "bck")
})
})
})
}
func populateExpansions(d *distro.Distro, bv *BuildVariant, t *task.Task) *command.Expansions {
expansions := command.NewExpansions(map[string]string{})
expansions.Put("execution", fmt.Sprintf("%v", t.Execution))
expansions.Put("version_id", t.Version)
expansions.Put("task_id", t.Id)
expansions.Put("task_name", t.DisplayName)
expansions.Put("build_id", t.BuildId)
expansions.Put("build_variant", t.BuildVariant)
expansions.Put("workdir", d.WorkDir)
expansions.Put("revision", t.Revision)
expansions.Put("project", t.Project)
expansions.Put("branch_name", t.Project)
if t.Requester == evergreen.PatchVersionRequester {
expansions.Put("is_patch", "true")
}
for _, e := range d.Expansions {
expansions.Put(e.Key, e.Value)
}
expansions.Update(bv.Expansions)
return expansions
}
// Creates a copy of `current`, with instances of ${matrixParameterName} in
// top level string and slice of strings fields replaced by the value specified
// in matrixParameterValues
func expandBuildVariantMatrixParameters(project *Project, current BuildVariant,
matrixParameterValues []MatrixParameterValue) (*BuildVariant, error) {
// Create a new build variant with the same parameters
newBv := current
newBv.Expansions = make(map[string]string)
// Make sure to copy over expansions
for k, v := range current.Expansions {
newBv.Expansions[k] = v
}
// Convert parameter value state into a map for use with expansions
matrixParameterMap := make(map[string]string)
for i, parameter := range project.BuildVariantMatrix.MatrixParameters {
matrixParameterMap[parameter.Name] = matrixParameterValues[i].Value
}
matrixParameterExpansions := command.NewExpansions(matrixParameterMap)
// Iterate over all fields
numFields := reflect.TypeOf(newBv).NumField()
for fieldIndex := 0; fieldIndex < numFields; fieldIndex++ {
// Expand matrix parameters in top level string fields
if reflect.TypeOf(newBv).Field(fieldIndex).Type.Kind() == reflect.String {
val := reflect.ValueOf(&newBv).Elem().Field(fieldIndex).String()
val, err := matrixParameterExpansions.ExpandString(val)
if err != nil {
return nil, err
}
reflect.ValueOf(&newBv).Elem().Field(fieldIndex).SetString(val)
}
// Expand matrix parameters in top level slices of strings
if reflect.TypeOf(newBv).Field(fieldIndex).Type == reflect.SliceOf(reflect.TypeOf("")) {
slice := reflect.ValueOf(&newBv).Elem().Field(fieldIndex)
newSlice := []string{}
for arrayIndex := 0; arrayIndex < slice.Len(); arrayIndex++ {
// Expand matrix parameters for each individual element of the slice
val := slice.Index(arrayIndex).String()
val, err := matrixParameterExpansions.ExpandString(val)
if err != nil {
return nil, err
}
newSlice = append(newSlice, val)
}
reflect.ValueOf(&newBv).Elem().Field(fieldIndex).Set(reflect.ValueOf(newSlice))
}
}
// First, attach all conditional expansions (i.e. expansions associated with
// a given parameter value)
for _, value := range matrixParameterValues {
for k, v := range value.Expansions {
newBv.Expansions[k] = v
}
}
// Then, expand matrix parameters in all expansions
for key, expansion := range newBv.Expansions {
expansion, err := matrixParameterExpansions.ExpandString(expansion)
if err != nil {
return nil, err
}
newBv.Expansions[key] = expansion
}
// Build variant matrix parameter values are stored in the build variant
buildVariantMatrixParameterValues := make(map[string]string)
for i, matrixParameter := range project.BuildVariantMatrix.MatrixParameters {
buildVariantMatrixParameterValues[matrixParameter.Name] =
matrixParameterValues[i].Value
}
newBv.MatrixParameterValues = buildVariantMatrixParameterValues
return &newBv, nil
}
// CreateHost spawns a host with the given options.
func (sm Spawn) CreateHost(so Options) (*host.Host, error) {
// load in the appropriate distro
d, err := distro.FindOne(distro.ById(so.Distro))
if err != nil {
return nil, err
}
// get the appropriate cloud manager
cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings)
if err != nil {
return nil, err
}
// spawn the host
h, err := cloudManager.SpawnInstance(d, so.UserName, true)
if err != nil {
return nil, err
}
// set the expiration time for the host
expireTime := h.CreationTime.Add(DefaultExpiration)
err = h.SetExpirationTime(expireTime)
if err != nil {
return h, evergreen.Logger.Errorf(slogger.ERROR,
"error setting expiration on host %v: %v", h.Id, err)
}
// set the user data, if applicable
if so.UserData != "" {
err = h.SetUserData(so.UserData)
if err != nil {
return h, evergreen.Logger.Errorf(slogger.ERROR,
"Failed setting userData on host %v: %v", h.Id, err)
}
}
// create a hostinit to take care of setting up the host
init := &hostinit.HostInit{
Settings: sm.settings,
}
// for making sure the host doesn't take too long to spawn
startTime := time.Now()
// spin until the host is ready for its setup script to be run
for {
// make sure we haven't been spinning for too long
if time.Now().Sub(startTime) > 15*time.Minute {
if err := h.SetDecommissioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err)
}
return nil, fmt.Errorf("host took too long to come up")
}
time.Sleep(5000 * time.Millisecond)
evergreen.Logger.Logf(slogger.INFO, "Checking if host %v is up and ready", h.Id)
// see if the host is ready for its setup script to be run
ready, err := init.IsHostReady(h)
if err != nil {
if err := h.SetDecommissioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err)
}
return nil, fmt.Errorf("error checking on host %v; decommissioning to save resources: %v",
h.Id, err)
}
// if the host is ready, move on to running the setup script
if ready {
break
}
}
evergreen.Logger.Logf(slogger.INFO, "Host %v is ready for its setup script to be run", h.Id)
// add any extra user-specified data into the setup script
if h.Distro.UserData.File != "" {
userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n",
strings.Replace(so.UserData, "\"", "\\\"", -1), h.Distro.UserData.File)
// prepend the setup script to add the userdata file
if strings.HasPrefix(h.Distro.Setup, "#!") {
firstLF := strings.Index(h.Distro.Setup, "\n")
h.Distro.Setup = h.Distro.Setup[0:firstLF+1] + userDataCmd + h.Distro.Setup[firstLF+1:]
} else {
h.Distro.Setup = userDataCmd + h.Distro.Setup
}
}
// modify the setup script to add the user's public key
h.Distro.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n",
so.PublicKey, h.Distro.User)
// replace expansions in the script
exp := command.NewExpansions(init.Settings.Expansions)
h.Distro.Setup, err = exp.ExpandString(h.Distro.Setup)
if err != nil {
return nil, fmt.Errorf("expansions error: %v", err)
}
// provision the host
err = init.ProvisionHost(h)
if err != nil {
return nil, fmt.Errorf("error provisioning host %v: %v", h.Id, err)
}
return h, nil
}
func TestExpandValues(t *testing.T) {
Convey("When expanding struct values", t, func() {
expansions := command.NewExpansions(
map[string]string{
"exp1": "val1",
},
)
Convey("if the input value is not a pointer to a struct, an error"+
" should be returned", func() {
So(ExpandValues("hello", expansions), ShouldNotBeNil)
So(ExpandValues(struct{}{}, expansions), ShouldNotBeNil)
So(ExpandValues([]string{"hi"}, expansions), ShouldNotBeNil)
})
Convey("if any non-string fields are tagged as expandable, an error"+
" should be returned", func() {
type s struct {
FieldOne string `plugin:"expand"`
FieldTwo int `plugin:"expand"`
}
So(ExpandValues(&s{}, expansions), ShouldNotBeNil)
})
Convey("any fields of the input struct with the appropriate tag should"+
" be expanded", func() {
type s struct {
FieldOne string
FieldTwo string `plugin:"expand"`
FieldThree string `plugin:"expand,hello"`
}
s1 := &s{
FieldOne: "hello ${exp1}",
FieldTwo: "hi ${exp1}",
FieldThree: "yo ${exp2|yo}",
}
So(ExpandValues(s1, expansions), ShouldBeNil)
// make sure the appropriate fields were expanded
So(s1.FieldOne, ShouldEqual, "hello ${exp1}")
So(s1.FieldTwo, ShouldEqual, "hi val1")
So(s1.FieldThree, ShouldEqual, "yo yo")
})
Convey("any nested structs tagged as expandable should have their"+
" fields expanded appropriately", func() {
type inner struct {
FieldOne string `plugin:"expand"`
}
type outer struct {
FieldOne string `plugin:"expand"`
FieldTwo inner `plugin:"expand"`
FieldThree inner
}
s := &outer{
FieldOne: "hello ${exp1}",
FieldTwo: inner{
FieldOne: "hi ${exp1}",
},
FieldThree: inner{
FieldOne: "yo ${exp1}",
},
}
So(ExpandValues(s, expansions), ShouldBeNil)
// make sure all fields, including nested ones, were expanded
// correctly
So(s.FieldOne, ShouldEqual, "hello val1")
So(s.FieldTwo.FieldOne, ShouldEqual, "hi val1")
So(s.FieldThree.FieldOne, ShouldEqual, "yo ${exp1}")
})
Convey("any nested maps tagged as expandable should have their"+
" fields expanded appropriately", func() {
type outer struct {
FieldOne string `plugin:"expand"`
FieldTwo map[string]string `plugin:"expand"`
}
s := &outer{
FieldOne: "hello ${exp1}",
FieldTwo: map[string]string{
"1": "hi ${exp1}",
"${exp1}": "yo ${exp1}",
},
}
So(ExpandValues(s, expansions), ShouldBeNil)
// make sure all fields, including nested maps, were expanded
So(s.FieldOne, ShouldEqual, "hello val1")
So(s.FieldTwo["1"], ShouldEqual, "hi val1")
So(s.FieldTwo["val1"], ShouldEqual, "yo val1")
})
Convey("if the input value is a slice, expansion should work "+
"for the fields within that slice", func() {
type simpleStruct struct {
StructFieldKeyOne string `plugin:"expand"`
StructFieldKeyTwo string
}
type sliceStruct struct {
SliceFieldKey []*simpleStruct `plugin:"expand"`
}
simpleStruct1 := simpleStruct{
StructFieldKeyOne: "hello ${exp1}",
StructFieldKeyTwo: "abc${expl}",
}
simpleSlice := make([]*simpleStruct, 0)
simpleSlice = append(simpleSlice, &simpleStruct1)
sliceStruct1 := sliceStruct{}
sliceStruct1.SliceFieldKey = simpleSlice
So(ExpandValues(&sliceStruct1, expansions), ShouldBeNil)
// make sure the appropriate fields were expanded
So(simpleStruct1.StructFieldKeyOne, ShouldEqual, "hello val1")
So(simpleStruct1.StructFieldKeyTwo, ShouldEqual, "abc${expl}")
})
Convey("any nested structs/slices tagged as expandable should have "+
"their fields expanded appropriately", func() {
type innerStruct struct {
FieldOne string `plugin:"expand"`
}
type innerSlice struct {
FieldOne string `plugin:"expand"`
}
type middle struct {
FieldOne string `plugin:"expand"`
FieldTwo string
FieldThree innerStruct
FieldFour []*innerSlice
}
type outer struct {
FieldOne string `plugin:"expand"`
FieldTwo []*middle `plugin:"expand"`
}
innerStructObject := innerStruct{
FieldOne: "hello ${exp1}",
}
innerSliceField := innerSlice{
FieldOne: "hi ${exp1}",
}
innerSliceObject := make([]*innerSlice, 0)
innerSliceObject = append(innerSliceObject, &innerSliceField)
middleObjectOne := middle{
FieldOne: "ab ${exp1}",
FieldTwo: "abc ${exp1}",
}
middleObjectTwo := middle{
FieldOne: "abc ${exp1}",
FieldTwo: "abc ${exp1}",
FieldThree: innerStructObject,
FieldFour: innerSliceObject,
}
middleObject := make([]*middle, 0)
middleObject = append(middleObject, &middleObjectOne)
middleObject = append(middleObject, &middleObjectTwo)
s := &outer{
FieldOne: "hello ${exp1}",
FieldTwo: middleObject,
}
So(ExpandValues(s, expansions), ShouldBeNil)
// make sure all fields, including nested ones, were expanded
// correctly
So(s.FieldOne, ShouldEqual, "hello val1")
So(s.FieldTwo[0].FieldOne, ShouldEqual, "ab val1")
So(s.FieldTwo[1].FieldOne, ShouldEqual, "abc val1")
So(s.FieldTwo[0].FieldTwo, ShouldEqual, "abc ${exp1}")
So(s.FieldTwo[1].FieldTwo, ShouldEqual, "abc ${exp1}")
So(s.FieldTwo[1].FieldThree.FieldOne, ShouldEqual, "hello ${exp1}")
So(s.FieldTwo[1].FieldFour[0].FieldOne, ShouldEqual, "hi ${exp1}")
})
})
Convey("When expanding map values", t, func() {
expansions := command.NewExpansions(
map[string]string{
"a": "A",
"b": "B",
"c": "C",
},
)
Convey("a simple map expands properly", func() {
testmap := map[string]string{
"nope": "nothing",
"${a}": "key",
"val": "${b}",
"${c}": "${a}",
}
So(ExpandValues(&testmap, expansions), ShouldBeNil)
So(testmap["nope"], ShouldEqual, "nothing")
So(testmap["A"], ShouldEqual, "key")
So(testmap["val"], ShouldEqual, "B")
So(testmap["C"], ShouldEqual, "A")
})
Convey("a recursive map expands properly", func() {
testmap := map[string]map[string]string{
"${a}": {
"deep": "${c}",
"no": "same",
},
}
So(ExpandValues(&testmap, expansions), ShouldBeNil)
So(len(testmap), ShouldEqual, 1)
So(len(testmap["A"]), ShouldEqual, 2)
So(testmap["A"]["no"], ShouldEqual, "same")
So(testmap["A"]["deep"], ShouldEqual, "C")
})
})
}
func TestLocalJob(t *testing.T) {
Convey("With an agent command", t, func() {
Convey("command's stdout/stderr should be captured by logger", func() {
appender := &evergreen.SliceAppender{[]*slogger.Log{}}
killChan := make(chan bool)
testCmd := &AgentCommand{
ScriptLine: "echo 'hi stdout!'; echo 'hi stderr!' >&2;",
StreamLogger: testutil.NewTestLogger(appender),
KillChan: killChan,
Expansions: command.NewExpansions(map[string]string{}),
}
err := testCmd.Run("")
So(err, ShouldBeNil)
testCmd.FlushAndWait()
// 2 lines from the command, plus 2 lines from the Run() func itself
for _, v := range appender.Messages {
fmt.Println(v.Message())
}
var levelToString = map[slogger.Level]string{
slogger.ERROR: "hi stderr!",
slogger.INFO: "hi stdout!",
}
So(len(appender.Messages), ShouldEqual, 4)
MsgA := appender.Messages[len(appender.Messages)-2]
MsgB := appender.Messages[len(appender.Messages)-1]
So(MsgA.Message(), ShouldEndWith, levelToString[MsgA.Level])
So(MsgB.Message(), ShouldEndWith, levelToString[MsgB.Level])
})
Convey("command's stdout/stderr should only print newlines with \\n", func() {
appender := &evergreen.SliceAppender{[]*slogger.Log{}}
killChan := make(chan bool)
newlineTestCmd := &AgentCommand{
ScriptLine: "printf 'this is not a newline...'; printf 'this is a newline \n';",
StreamLogger: testutil.NewTestLogger(appender),
KillChan: killChan,
Expansions: command.NewExpansions(map[string]string{}),
}
err := newlineTestCmd.Run("")
So(err, ShouldBeNil)
newlineTestCmd.FlushAndWait()
// 2 lines from the command, plus 1 lines from the Run() func itself
for _, v := range appender.Messages {
fmt.Println(v.Message())
}
So(len(appender.Messages), ShouldEqual, 3)
NewLineMessage := appender.Messages[len(appender.Messages)-1]
So(NewLineMessage.Message(), ShouldEqual, "this is not a newline...this is a newline ")
})
})
Convey("With a long-running agent command", t, func() {
appender := &evergreen.SliceAppender{[]*slogger.Log{}}
killChan := make(chan bool)
testCmd := &AgentCommand{
ScriptLine: "echo 'hi'; sleep 4; echo 'i should not get run'",
StreamLogger: testutil.NewTestLogger(appender),
KillChan: killChan,
Expansions: command.NewExpansions(map[string]string{}),
}
Convey("using kill channel should abort command right away", func() {
commandChan := make(chan error)
go func() {
err := testCmd.Run("")
commandChan <- err
}()
go func() {
// after a delay, signal the command to stop
time.Sleep(1 * time.Second)
close(killChan)
}()
err := <-commandChan
So(err, ShouldEqual, InterruptedCmdError)
testCmd.Flush()
lastMessage := appender.Messages[len(appender.Messages)-1]
nextLastMessage := appender.Messages[len(appender.Messages)-2]
So(lastMessage.Message(), ShouldStartWith, "Got kill signal")
So(nextLastMessage.Message(), ShouldEqual, "hi")
})
})
}
// CreateHost spawns a host with the given options.
func (sm Spawn) CreateHost(so Options, owner *user.DBUser) error {
// load in the appropriate distro
d, err := distro.FindOne(distro.ById(so.Distro))
if err != nil {
return err
}
// add any extra user-specified data into the setup script
if d.UserData.File != "" {
userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n",
strings.Replace(so.UserData, "\"", "\\\"", -1), d.UserData.File)
// prepend the setup script to add the userdata file
if strings.HasPrefix(d.Setup, "#!") {
firstLF := strings.Index(d.Setup, "\n")
d.Setup = d.Setup[0:firstLF+1] + userDataCmd + d.Setup[firstLF+1:]
} else {
d.Setup = userDataCmd + d.Setup
}
}
// modify the setup script to add the user's public key
d.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n", so.PublicKey, d.User)
// replace expansions in the script
exp := command.NewExpansions(sm.settings.Expansions)
d.Setup, err = exp.ExpandString(d.Setup)
if err != nil {
return fmt.Errorf("expansions error: %v", err)
}
// fake out replacing spot instances with on-demand equivalents
if d.Provider == ec2.SpotProviderName {
d.Provider = ec2.OnDemandProviderName
}
// get the appropriate cloud manager
cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings)
if err != nil {
return err
}
// spawn the host
provisionOptions := &host.ProvisionOptions{
LoadCLI: true,
TaskId: so.TaskId,
OwnerId: owner.Id,
}
expiration := DefaultExpiration
hostOptions := cloud.HostOptions{
ProvisionOptions: provisionOptions,
UserName: so.UserName,
ExpirationDuration: &expiration,
UserData: so.UserData,
UserHost: true,
}
_, err = cloudManager.SpawnInstance(d, hostOptions)
if err != nil {
return err
}
return nil
}