func testProcessCancelCommandMessage(t *testing.T, testCase TestCaseCancelCommand) { context := context.NewMockDefault() // create a cancel message cancelMessagePayload := messageContracts.CancelPayload{ CancelMessageID: "aws.ssm" + testCase.MsgToCancelID + "." + testCase.InstanceID, } msgContent, err := jsonutil.Marshal(cancelMessagePayload) if err != nil { t.Fatal(err) } mdsCancelMessage := createMDSMessage(testCase.MsgID, msgContent, "aws.ssm.cancelCommand.us.east.1.1", testCase.InstanceID) // method should call the proper APIs on the MDS service mdsMock := new(MockedMDS) mdsMock.On("DeleteMessage", mock.Anything, *mdsCancelMessage.MessageId).Return(nil) // method should call cancel command sendCommandPoolMock := new(task.MockedPool) sendCommandPoolMock.On("Cancel", cancelMessagePayload.CancelMessageID).Return(true) p := Processor{} // call the code we are testing p.processCancelCommandMessage(context, mdsMock, sendCommandPoolMock, mdsCancelMessage) // assert that the expectations were met mdsMock.AssertExpectations(t) sendCommandPoolMock.AssertExpectations(t) }
func generateTestCaseFromFiles(t *testing.T, messagePayloadFile string, messageReplyPayloadFile string, instanceID string) (testCase TestCaseSendCommand) { // load message payload and create MDS message from it payload, err := parser.ParseMessageWithParams(logger, string(loadFile(t, messagePayloadFile))) if err != nil { t.Fatal(err) } msgContent, err := jsonutil.Marshal(payload) if err != nil { t.Fatal(err) } testCase.Msg = createMDSMessage(payload.CommandID, msgContent, "aws.ssm.sendCommand.us.east.1.1", instanceID) testCase.MsgPayload = payload s3KeyPrefix := path.Join(payload.OutputS3KeyPrefix, payload.CommandID, *testCase.Msg.Destination) //orchestrationRootDir is set to CommandID considering that orchestration root directory name will be empty in the test case. orchestrationRootDir := getCommandID(*testCase.Msg.MessageId) testCase.PluginConfigs = getPluginConfigurations(payload.DocumentContent.RuntimeConfig, orchestrationRootDir, payload.OutputS3BucketName, s3KeyPrefix, *testCase.Msg.MessageId) testCase.PluginResults = make(map[string]*contracts.PluginResult) testCase.ReplyPayload = loadMessageReplyFromFile(t, messageReplyPayloadFile) for pluginName, pluginRuntimeStatus := range testCase.ReplyPayload.RuntimeStatus { pluginResult := parsePluginResult(t, *pluginRuntimeStatus) testCase.PluginResults[pluginName] = &pluginResult } return }
// Our intent here is to look for array of commands which will be executed as a part of this document and replace the incompatible code. func removeDependencyOnInstanceMetadataForManagedInstance(context context.T, parsedMessage messageContracts.SendCommandPayload) (messageContracts.SendCommandPayload, error) { log := context.Log() var properties []interface{} var parsedDocumentProperties ManagedInstanceDocumentProperties err := jsonutil.Remarshal(parsedMessage.DocumentContent.RuntimeConfig[appconfig.PluginNameAwsRunScript].Properties, &properties) if err != nil { log.Errorf("Invalid format of properties in %v document. error: %v", parsedMessage.DocumentName, err) return parsedMessage, err } // Since 'Properties' is an array and we use only one property block for the above documents, array location '0' of 'Properties' is used. err = jsonutil.Remarshal(properties[0], &parsedDocumentProperties) if err != nil { log.Errorf("Invalid format of properties in %v document. error: %v", parsedMessage.DocumentName, err) return parsedMessage, err } region, err := platform.Region() if err != nil { log.Errorf("Error retrieving agent region. error: %v", err) return parsedMessage, err } // Comment or replace the incompatible code from this document. log.Info("Replacing managed instance incompatible code for AWS SSM Document.") for i, command := range parsedDocumentProperties.RunCommand { if strings.Contains(command, "$metadataLocation = 'http://169.254.169.254/latest/dynamic/instance-identity/document/region'") { parsedDocumentProperties.RunCommand[i] = strings.Replace(command, "$metadataLocation = 'http://169.254.169.254/latest/dynamic/instance-identity/document/region'", "# $metadataLocation = 'http://169.254.169.254/latest/dynamic/instance-identity/document/region' (This is done to make it managed instance compatible)", 1) } if strings.Contains(command, "$metadata = (New-Object Net.WebClient).DownloadString($metadataLocation)") { parsedDocumentProperties.RunCommand[i] = strings.Replace(command, "$metadata = (New-Object Net.WebClient).DownloadString($metadataLocation)", "# $metadata = (New-Object Net.WebClient).DownloadString($metadataLocation) (This is done to make it managed instance compatible)", 1) } if strings.Contains(command, "$region = (ConvertFrom-JSON $metadata).region") { parsedDocumentProperties.RunCommand[i] = strings.Replace(command, "$region = (ConvertFrom-JSON $metadata).region", "$region = '"+region+"'", 1) } } // Plug-in the compatible 'Properties' block back to the document. properties[0] = parsedDocumentProperties var documentProperties interface{} = properties parsedMessage.DocumentContent.RuntimeConfig[appconfig.PluginNameAwsRunScript].Properties = documentProperties // For debug purposes. parsedMessageContent, err := jsonutil.Marshal(parsedMessage) if err != nil { log.Errorf("Error marshalling %v document. error: %v", parsedMessage.DocumentName, err) return parsedMessage, err } log.Debug("ParsedMessage after removing dependency on instance metadata for managed instance is ", jsonutil.Indent(parsedMessageContent)) return parsedMessage, nil }
// setCmdState persists given commandState func setCmdState(log log.T, commandState message.CommandState, absoluteFileName, locationFolder string) { content, err := jsonutil.Marshal(commandState) if err != nil { log.Errorf("encountered error with message %v while marshalling %v to string", err, commandState) } else { if fileutil.Exists(absoluteFileName) { log.Debugf("overwriting contents of %v", absoluteFileName) } log.Tracef("persisting interim state %v in file %v", jsonutil.Indent(content), absoluteFileName) if s, err := fileutil.WriteIntoFileWithPermissions(absoluteFileName, jsonutil.Indent(content), os.FileMode(int(appconfig.ReadWriteAccess))); s && err == nil { log.Debugf("successfully persisted interim state in %v", locationFolder) } else { log.Debugf("persisting interim state in %v failed with error %v", locationFolder, err) } } }
// getCmdState reads commandState from given file func getCmdState(log log.T, fileName string) message.CommandState { var commandState message.CommandState err := jsonutil.UnmarshalFile(fileName, &commandState) if err != nil { log.Errorf("encountered error with message %v while reading Interim state of command from file - %v", err, fileName) } else { //logging interim state as read from the file jsonString, err := jsonutil.Marshal(commandState) if err != nil { log.Errorf("encountered error with message %v while marshalling %v to string", err, commandState) } else { log.Tracef("interim CommandState read from file-system - %v", jsonutil.Indent(jsonString)) } } return commandState }
// PersistData stores the given object in the file-system in pretty Json indented format // This will override the contents of an already existing file func PersistData(log log.T, commandID, instanceID, locationFolder string, object interface{}) { lockDocument(commandID) defer unlockDocument(commandID) absoluteFileName := getCmdStateFileName(commandID, instanceID, locationFolder) content, err := jsonutil.Marshal(object) if err != nil { log.Errorf("encountered error with message %v while marshalling %v to string", err, object) } else { if fileutil.Exists(absoluteFileName) { log.Debugf("overwriting contents of %v", absoluteFileName) } log.Tracef("persisting interim state %v in file %v", jsonutil.Indent(content), absoluteFileName) if s, err := fileutil.WriteIntoFileWithPermissions(absoluteFileName, jsonutil.Indent(content), os.FileMode(int(appconfig.ReadWriteAccess))); s && err == nil { log.Debugf("successfully persisted interim state in %v", locationFolder) } else { log.Debugf("persisting interim state in %v failed with error %v", locationFolder, err) } } }
// runCommands executes one set of commands and returns their output. func (p *Plugin) runCommands(log log.T, pluginInput ApplicationPluginInput, orchestrationDirectory string, cancelFlag task.CancelFlag, outputS3BucketName string, outputS3KeyPrefix string) (out ApplicationPluginOutput) { var err error // if no orchestration directory specified, create temp directory var useTempDirectory = (orchestrationDirectory == "") var tempDir string if useTempDirectory { if tempDir, err = ioutil.TempDir("", "Ec2RunCommand"); err != nil { out.Errors = append(out.Errors, err.Error()) log.Error(err) return } orchestrationDirectory = tempDir } orchestrationDir := fileutil.RemoveInvalidChars(filepath.Join(orchestrationDirectory, pluginInput.ID)) log.Debugf("OrchestrationDir %v ", orchestrationDir) // create orchestration dir if needed if err = fileutil.MakeDirs(orchestrationDir); err != nil { log.Debug("failed to create orchestrationDir directory", orchestrationDir, err) out.Errors = append(out.Errors, err.Error()) return } // Get application mode mode, err := getMsiApplicationMode(pluginInput) if err != nil { out.MarkAsFailed(log, err) return } log.Debugf("mode is %v", mode) // Download file from source if available downloadOutput, err := pluginutil.DownloadFileFromSource(log, pluginInput.Source, pluginInput.SourceHash, pluginInput.SourceHashType) if err != nil || downloadOutput.IsHashMatched == false || downloadOutput.LocalFilePath == "" { errorString := fmt.Errorf("failed to download file reliably %v", pluginInput.Source) out.MarkAsFailed(log, errorString) return } log.Debugf("local path to file is %v", downloadOutput.LocalFilePath) // Create msi related log file localSourceLogFilePath := downloadOutput.LocalFilePath + ".msiexec.log.txt" log.Debugf("log path is %v", localSourceLogFilePath) // TODO: This needs to be pulled out of this function as it runs multiple times getting initialized with the same values // Create output file paths stdoutFilePath := filepath.Join(orchestrationDir, p.StdoutFileName) stderrFilePath := filepath.Join(orchestrationDir, p.StderrFileName) log.Debugf("stdout file %v, stderr file %v", stdoutFilePath, stderrFilePath) // Construct Command Name and Arguments commandName := msiExecCommand commandArguments := []string{mode, downloadOutput.LocalFilePath, "/quiet", "/norestart", "/log", localSourceLogFilePath} if pluginInput.Parameters != "" { log.Debugf("Got Parameters \"%v\"", pluginInput.Parameters) params := processParams(log, pluginInput.Parameters) commandArguments = append(commandArguments, params...) } // Execute Command _, _, exitCode, errs := p.ExecuteCommand(log, defaultWorkingDirectory, stdoutFilePath, stderrFilePath, cancelFlag, defaultApplicationExecutionTimeoutInSeconds, commandName, commandArguments) // Set output status out.ExitCode = exitCode setMsiExecStatus(log, pluginInput, cancelFlag, &out) if len(errs) > 0 { for _, err := range errs { out.Errors = append(out.Errors, err.Error()) log.Error("failed to run commands: ", err) out.Status = contracts.ResultStatusFailed } return } // Upload output to S3 uploadOutputToS3BucketErrors := p.ExecuteUploadOutputToS3Bucket(log, pluginInput.ID, orchestrationDir, outputS3BucketName, outputS3KeyPrefix, useTempDirectory, tempDir, out.Stdout, out.Stderr) out.Errors = append(out.Errors, uploadOutputToS3BucketErrors...) // Return Json indented response responseContent, _ := jsonutil.Marshal(out) log.Debug("Returning response:\n", jsonutil.Indent(responseContent)) return }
// processSendCommandMessage processes a single send command message received from MDS. func (p *Processor) processSendCommandMessage(context context.T, mdsService service.Service, messagesOrchestrationRootDir string, runPlugins PluginRunner, cancelFlag task.CancelFlag, buildReply replyBuilder, sendResponse engine.SendResponse, msg ssmmds.Message) { commandID := getCommandID(*msg.MessageId) log := context.Log() log.Debug("Processing send command message ", *msg.MessageId) log.Trace("Processing send command message ", jsonutil.Indent(*msg.Payload)) parsedMessage, err := parser.ParseMessageWithParams(log, *msg.Payload) if err != nil { log.Error("format of received message is invalid ", err) err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } return } parsedMessageContent, _ := jsonutil.Marshal(parsedMessage) log.Debug("ParsedMessage is ", jsonutil.Indent(parsedMessageContent)) // adapt plugin configuration format from MDS to plugin expected format s3KeyPrefix := path.Join(parsedMessage.OutputS3KeyPrefix, parsedMessage.CommandID, *msg.Destination) messageOrchestrationDirectory := filepath.Join(messagesOrchestrationRootDir, commandID) // Check if it is a managed instance and its executing managed instance incompatible AWS SSM public document. // A few public AWS SSM documents contain code which is not compatible when run on managed instances. // isManagedInstanceIncompatibleAWSSSMDocument makes sure to find such documents at runtime and replace the incompatible code. isMI, err := isManagedInstance() if err != nil { log.Errorf("Error determining managed instance. error: %v", err) } if isMI && isManagedInstanceIncompatibleAWSSSMDocument(parsedMessage.DocumentName) { log.Debugf("Running Incompatible AWS SSM Document %v on managed instance", parsedMessage.DocumentName) if parsedMessage, err = removeDependencyOnInstanceMetadataForManagedInstance(context, parsedMessage); err != nil { return } } pluginConfigurations := getPluginConfigurations( parsedMessage.DocumentContent.RuntimeConfig, messageOrchestrationDirectory, parsedMessage.OutputS3BucketName, s3KeyPrefix, *msg.MessageId) //persist : all information in current folder log.Info("Persisting message in current execution folder") //Data format persisted in Current Folder is defined by the struct - CommandState interimCmdState := initializeCommandState(pluginConfigurations, msg, parsedMessage) // persist new interim command state in current folder commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, interimCmdState) //Deleting from pending folder since the command is getting executed commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending) log.Debug("Running plugins...") outputs := runPlugins(context, *msg.MessageId, pluginConfigurations, sendResponse, cancelFlag) pluginOutputContent, _ := jsonutil.Marshal(outputs) log.Debugf("Plugin outputs %v", jsonutil.Indent(pluginOutputContent)) payloadDoc := buildReply("", outputs) //check if document isn't supported by SSM -> update the DocumentLevel status message & send reply accordingly ssmDocName := parsedMessage.DocumentName if IsDocumentNotSupportedBySsmAgent(ssmDocName) { log.Infof("%s is not yet supported by aws-ssm-agent, setting up Document level response accordingly", ssmDocName) payloadDoc.DocumentTraceOutput = fmt.Sprintf("%s document is not yet supported by amazon-ssm-agent.", ssmDocName) p.sendDocLevelResponse(*msg.MessageId, contracts.ResultStatusFailed, payloadDoc.DocumentTraceOutput) } //update documentInfo in interim cmd state file documentInfo := commandStateHelper.GetDocumentInfo(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent) // set document level information which wasn't set previously documentInfo.AdditionalInfo = payloadDoc.AdditionalInfo documentInfo.DocumentStatus = payloadDoc.DocumentStatus documentInfo.DocumentTraceOutput = payloadDoc.DocumentTraceOutput documentInfo.RuntimeStatus = payloadDoc.RuntimeStatus //persist final documentInfo. commandStateHelper.PersistDocumentInfo(log, documentInfo, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent) // Skip sending response when the document requires a reboot if documentInfo.DocumentStatus == contracts.ResultStatusSuccessAndReboot { log.Debug("skipping sending response of %v since the document requires a reboot", *msg.MessageId) return } log.Debug("Sending reply on message completion ", outputs) sendResponse(*msg.MessageId, "", outputs) //persist : commands execution in completed folder (terminal state folder) log.Debugf("execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId) commandStateHelper.MoveCommandState(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, appconfig.DefaultLocationOfCompleted) log.Debugf("Deleting message") isUpdate := false for pluginName := range pluginConfigurations { if pluginName == appconfig.PluginNameAwsAgentUpdate { isUpdate = true } } if !isUpdate { err = mdsService.DeleteMessage(log, *msg.MessageId) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } } else { log.Debug("MessageDeletion skipped as it will be handled by external process") } }
// runCmdsUsingCmdState takes commandState as an input and executes only those plugins which haven't yet executed. This is functionally // very similar to processSendCommandMessage because everything to do with cmd execution is part of that function right now. func (p *Processor) runCmdsUsingCmdState(context context.T, mdsService service.Service, runPlugins PluginRunner, cancelFlag task.CancelFlag, buildReply replyBuilder, sendResponse engine.SendResponse, command messageContracts.CommandState) { log := context.Log() var pluginConfigurations map[string]*contracts.Configuration pendingPlugins := false pluginConfigurations = make(map[string]*contracts.Configuration) //iterate through all plugins to find all plugins that haven't executed yet. for k, v := range command.PluginsInformation { if v.HasExecuted { log.Debugf("skipping execution of Plugin - %v of command - %v since it has already executed.", k, command.DocumentInformation.CommandID) } else { log.Debugf("Plugin - %v of command - %v will be executed", k, command.DocumentInformation.CommandID) pluginConfigurations[k] = &v.Configuration pendingPlugins = true } } //execute plugins that haven't been executed yet //individual plugins after execution will update interim cmd state file accordingly if pendingPlugins { log.Debugf("executing following plugins of command - %v", command.DocumentInformation.CommandID) for k := range pluginConfigurations { log.Debugf("Plugin: %v", k) } //Since only some plugins of a cmd gets executed here - there is no need to get output from engine & construct the sendReply output. //Instead after all plugins of a command get executed, use persisted data to construct sendReply payload runPlugins(context, command.DocumentInformation.MessageID, pluginConfigurations, sendResponse, cancelFlag) } //read from persisted file newCmdState := commandStateHelper.GetCommandInterimState(log, command.DocumentInformation.CommandID, command.DocumentInformation.Destination, appconfig.DefaultLocationOfCurrent) //construct sendReply payload outputs := make(map[string]*contracts.PluginResult) for k, v := range newCmdState.PluginsInformation { outputs[k] = &v.Result } pluginOutputContent, _ := jsonutil.Marshal(outputs) log.Debugf("plugin outputs %v", jsonutil.Indent(pluginOutputContent)) payloadDoc := buildReply("", outputs) //update interim cmd state file with document level information var documentInfo messageContracts.DocumentInfo // set document level information which wasn't set previously documentInfo.AdditionalInfo = payloadDoc.AdditionalInfo documentInfo.DocumentStatus = payloadDoc.DocumentStatus documentInfo.DocumentTraceOutput = payloadDoc.DocumentTraceOutput documentInfo.RuntimeStatus = payloadDoc.RuntimeStatus //persist final documentInfo. commandStateHelper.PersistDocumentInfo(log, documentInfo, command.DocumentInformation.CommandID, command.DocumentInformation.Destination, appconfig.DefaultLocationOfCurrent) // Skip sending response when the document requires a reboot if documentInfo.DocumentStatus == contracts.ResultStatusSuccessAndReboot { log.Debug("skipping sending response of %v since the document requires a reboot", newCmdState.DocumentInformation.MessageID) return } //send document level reply log.Debug("sending reply on message completion ", outputs) sendResponse(command.DocumentInformation.MessageID, "", outputs) //persist : commands execution in completed folder (terminal state folder) log.Debugf("execution of %v is over. Moving interimState file from Current to Completed folder", newCmdState.DocumentInformation.MessageID) commandStateHelper.MoveCommandState(log, newCmdState.DocumentInformation.CommandID, newCmdState.DocumentInformation.Destination, appconfig.DefaultLocationOfCurrent, appconfig.DefaultLocationOfCompleted) log.Debugf("deleting message") isUpdate := false for pluginName := range pluginConfigurations { if pluginName == appconfig.PluginNameAwsAgentUpdate { isUpdate = true } } if !isUpdate { err := mdsService.DeleteMessage(log, newCmdState.DocumentInformation.MessageID) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } } else { log.Debug("messageDeletion skipped as it will be handled by external process") } }
// runCommands executes one set of commands and returns their output. func (p *Plugin) runCommands(log log.T, pluginInput PSModulePluginInput, orchestrationDirectory string, cancelFlag task.CancelFlag, outputS3BucketName string, outputS3KeyPrefix string) (out PSModulePluginOutput) { var err error // if no orchestration directory specified, create temp directory var useTempDirectory = (orchestrationDirectory == "") var tempDir string if useTempDirectory { if tempDir, err = ioutil.TempDir("", "Ec2RunCommand"); err != nil { out.Errors = append(out.Errors, err.Error()) log.Error(err) return } orchestrationDirectory = tempDir } orchestrationDir := fileutil.RemoveInvalidChars(filepath.Join(orchestrationDirectory, pluginInput.ID)) log.Debugf("Running commands %v in workingDirectory %v; orchestrationDir %v ", pluginInput.RunCommand, pluginInput.WorkingDirectory, orchestrationDir) // create orchestration dir if needed if err = fileutil.MakeDirsWithExecuteAccess(orchestrationDir); err != nil { log.Debug("failed to create orchestrationDir directory", orchestrationDir) out.Errors = append(out.Errors, err.Error()) return } // Create script file path scriptPath := filepath.Join(orchestrationDir, pluginutil.RunCommandScriptName) log.Debugf("Writing commands %v to file %v", pluginInput, scriptPath) // Create script file if err = pluginutil.CreateScriptFile(log, scriptPath, pluginInput.RunCommand); err != nil { out.Errors = append(out.Errors, err.Error()) log.Errorf("failed to create script file. %v", err) return } // Download file from source if available downloadOutput, err := pluginutil.DownloadFileFromSource(log, pluginInput.Source, pluginInput.SourceHash, pluginInput.SourceHashType) if err != nil || downloadOutput.IsHashMatched == false || downloadOutput.LocalFilePath == "" { errorString := fmt.Errorf("failed to download file reliably %v", pluginInput.Source) out.MarkAsFailed(log, errorString) return } else { // Uncompress the zip file received fileutil.Uncompress(downloadOutput.LocalFilePath, PowerShellModulesDirectory) } // Set execution time executionTimeout := pluginutil.ValidateExecutionTimeout(log, pluginInput.TimeoutSeconds) // Create output file paths stdoutFilePath := filepath.Join(orchestrationDir, p.StdoutFileName) stderrFilePath := filepath.Join(orchestrationDir, p.StderrFileName) log.Debugf("stdout file %v, stderr file %v", stdoutFilePath, stderrFilePath) // Construct Command Name and Arguments commandName := pluginutil.GetShellCommand() commandArguments := append(pluginutil.GetShellArguments(), scriptPath, pluginutil.ExitCodeTrap) // Execute Command stdout, stderr, exitCode, errs := p.ExecuteCommand(log, pluginInput.WorkingDirectory, stdoutFilePath, stderrFilePath, cancelFlag, executionTimeout, commandName, commandArguments) // Set output status out.ExitCode = exitCode out.Status = pluginutil.GetStatus(out.ExitCode, cancelFlag) if len(errs) > 0 { for _, err := range errs { out.Errors = append(out.Errors, err.Error()) if out.Status != contracts.ResultStatusCancelled && out.Status != contracts.ResultStatusTimedOut && out.Status != contracts.ResultStatusSuccessAndReboot { log.Error("failed to run commands: ", err) out.Status = contracts.ResultStatusFailed } } } // read (a prefix of) the standard output/error out.Stdout, err = pluginutil.ReadPrefix(stdout, p.MaxStdoutLength, p.OutputTruncatedSuffix) if err != nil { out.Errors = append(out.Errors, err.Error()) log.Error(err) } out.Stderr, err = pluginutil.ReadPrefix(stderr, p.MaxStderrLength, p.OutputTruncatedSuffix) if err != nil { out.Errors = append(out.Errors, err.Error()) log.Error(err) } // Upload output to S3 uploadOutputToS3BucketErrors := p.ExecuteUploadOutputToS3Bucket(log, pluginInput.ID, orchestrationDir, outputS3BucketName, outputS3KeyPrefix, useTempDirectory, tempDir, out.Stdout, out.Stderr) out.Errors = append(out.Errors, uploadOutputToS3BucketErrors...) // Return Json indented response responseContent, _ := jsonutil.Marshal(out) log.Debug("Returning response:\n", jsonutil.Indent(responseContent)) return }