// processCancelCommandMessage processes a single send command message received from MDS.
func (p *Processor) processCancelCommandMessage(context context.T,
mdsService service.Service,
sendCommandPool task.Pool,
msg ssmmds.Message) {
log := context.Log()
log.Debug("Processing cancel command message ", jsonutil.Indent(*msg.Payload))
var parsedMessage messageContracts.CancelPayload
err := json.Unmarshal([]byte(*msg.Payload), &parsedMessage)
if err != nil {
log.Error("format of received cancel message is invalid ", err)
err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException)
if err != nil {
sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
}
return
}
log.Debugf("ParsedMessage is %v", parsedMessage)
//persist in current folder here
cancelCmd := initializeCancelCommandState(msg, parsedMessage)
commandID := getCommandID(*msg.MessageId)
// persist new interim command state in current folder
commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, cancelCmd)
//remove from pending folder
commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending)
log.Debugf("Canceling job with id %v...", parsedMessage.CancelMessageID)
if found := sendCommandPool.Cancel(parsedMessage.CancelMessageID); !found {
log.Debugf("Job with id %v not found (possibly completed)", parsedMessage.CancelMessageID)
cancelCmd.DebugInfo = fmt.Sprintf("Command %v couldn't be cancelled", cancelCmd.CancelCommandID)
cancelCmd.Status = contracts.ResultStatusFailed
} else {
cancelCmd.DebugInfo = fmt.Sprintf("Command %v cancelled", cancelCmd.CancelCommandID)
cancelCmd.Status = contracts.ResultStatusSuccess
}
//persist the final status of cancel-message in current folder
commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, cancelCmd)
//persist : commands execution in completed folder (terminal state folder)
log.Debugf("Execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId)
commandStateHelper.MoveCommandState(log,
commandID,
*msg.Destination,
appconfig.DefaultLocationOfCurrent,
appconfig.DefaultLocationOfCompleted)
log.Debugf("Deleting message")
err = mdsService.DeleteMessage(log, *msg.MessageId)
if err != nil {
sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
}
}
// processOlderMessagesFromCurrent processes older messages that were persisted in CURRENT folder
func (p *Processor) processMessagesFromCurrent(instanceID string) {
log := p.context.Log()
config := p.context.AppConfig()
unprocessedMsgsLocation := path.Join(appconfig.DefaultDataStorePath,
instanceID,
appconfig.DefaultCommandRootDirName,
appconfig.DefaultLocationOfState,
appconfig.DefaultLocationOfCurrent)
if isDirectoryEmpty, _ := fileutil.IsDirEmpty(unprocessedMsgsLocation); isDirectoryEmpty {
log.Debugf("no older messages to process from %v", unprocessedMsgsLocation)
} else {
//get all older messages from previous run of agent
files, err := ioutil.ReadDir(unprocessedMsgsLocation)
//TODO: revisit this when bookkeeping is made invasive
if err != nil {
log.Errorf("skipping reading inprogress messages from %v. unexpected error encountered - %v", unprocessedMsgsLocation, err)
return
}
//iterate through all old executing messages
for _, f := range files {
log.Debugf("processing previously unexecuted message - %v", f.Name())
//construct the absolute path - safely assuming that interim state for older messages are already present in Current folder
file := path.Join(appconfig.DefaultDataStorePath,
instanceID,
appconfig.DefaultCommandRootDirName,
appconfig.DefaultLocationOfState,
appconfig.DefaultLocationOfCurrent,
f.Name())
var oldCmdState messageContracts.CommandState
//parse the message
//TODO: Not all messages in Current folder correspond to SendCommand.
if err := jsonutil.UnmarshalFile(file, &oldCmdState); err != nil {
log.Errorf("skipping processsing of previously unexecuted messages. encountered error %v while reading unprocessed message from file - %v", err, f)
} else {
if oldCmdState.DocumentInformation.RunCount >= config.Mds.CommandRetryLimit {
//TODO: Move command to corrupt/failed
// do not process as the command has failed too many times
break
}
pluginOutputs := make(map[string]*contracts.PluginResult)
// increment the command run count
oldCmdState.DocumentInformation.RunCount++
// Update reboot status
for v := range oldCmdState.PluginsInformation {
plugin := oldCmdState.PluginsInformation[v]
if plugin.HasExecuted && plugin.Result.Status == contracts.ResultStatusSuccessAndReboot {
log.Debugf("plugin %v has completed a reboot. Setting status to Success.", v)
plugin.Result.Status = contracts.ResultStatusSuccess
oldCmdState.PluginsInformation[v] = plugin
pluginOutputs[v] = &plugin.Result
p.sendResponse(oldCmdState.DocumentInformation.MessageID, v, pluginOutputs)
}
}
// func PersistData(log log.T, commandID, instanceID, locationFolder string, object interface{}) {
commandStateHelper.PersistData(log, oldCmdState.DocumentInformation.CommandID, instanceID, appconfig.DefaultLocationOfCurrent, oldCmdState)
//Submit the work to Job Pool so that we don't block for processing of new messages
err := p.sendCommandPool.Submit(log, oldCmdState.DocumentInformation.MessageID, func(cancelFlag task.CancelFlag) {
p.runCmdsUsingCmdState(p.context.With("[messageID="+oldCmdState.DocumentInformation.MessageID+"]"),
p.service,
p.pluginRunner,
cancelFlag,
p.buildReply,
p.sendResponse,
oldCmdState)
})
if err != nil {
log.Error("SendCommand failed for previously unexecuted commands", err)
return
}
}
}
}
}
// processSendCommandMessage processes a single send command message received from MDS.
func (p *Processor) processSendCommandMessage(context context.T,
mdsService service.Service,
messagesOrchestrationRootDir string,
runPlugins PluginRunner,
cancelFlag task.CancelFlag,
buildReply replyBuilder,
sendResponse engine.SendResponse,
msg ssmmds.Message) {
commandID := getCommandID(*msg.MessageId)
log := context.Log()
log.Debug("Processing send command message ", *msg.MessageId)
log.Trace("Processing send command message ", jsonutil.Indent(*msg.Payload))
parsedMessage, err := parser.ParseMessageWithParams(log, *msg.Payload)
if err != nil {
log.Error("format of received message is invalid ", err)
err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException)
if err != nil {
sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
}
return
}
parsedMessageContent, _ := jsonutil.Marshal(parsedMessage)
log.Debug("ParsedMessage is ", jsonutil.Indent(parsedMessageContent))
// adapt plugin configuration format from MDS to plugin expected format
s3KeyPrefix := path.Join(parsedMessage.OutputS3KeyPrefix, parsedMessage.CommandID, *msg.Destination)
messageOrchestrationDirectory := filepath.Join(messagesOrchestrationRootDir, commandID)
// Check if it is a managed instance and its executing managed instance incompatible AWS SSM public document.
// A few public AWS SSM documents contain code which is not compatible when run on managed instances.
// isManagedInstanceIncompatibleAWSSSMDocument makes sure to find such documents at runtime and replace the incompatible code.
isMI, err := isManagedInstance()
if err != nil {
log.Errorf("Error determining managed instance. error: %v", err)
}
if isMI && isManagedInstanceIncompatibleAWSSSMDocument(parsedMessage.DocumentName) {
log.Debugf("Running Incompatible AWS SSM Document %v on managed instance", parsedMessage.DocumentName)
if parsedMessage, err = removeDependencyOnInstanceMetadataForManagedInstance(context, parsedMessage); err != nil {
return
}
}
pluginConfigurations := getPluginConfigurations(
parsedMessage.DocumentContent.RuntimeConfig,
messageOrchestrationDirectory,
parsedMessage.OutputS3BucketName,
s3KeyPrefix,
*msg.MessageId)
//persist : all information in current folder
log.Info("Persisting message in current execution folder")
//Data format persisted in Current Folder is defined by the struct - CommandState
interimCmdState := initializeCommandState(pluginConfigurations, msg, parsedMessage)
// persist new interim command state in current folder
commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, interimCmdState)
//Deleting from pending folder since the command is getting executed
commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending)
log.Debug("Running plugins...")
outputs := runPlugins(context, *msg.MessageId, pluginConfigurations, sendResponse, cancelFlag)
pluginOutputContent, _ := jsonutil.Marshal(outputs)
log.Debugf("Plugin outputs %v", jsonutil.Indent(pluginOutputContent))
payloadDoc := buildReply("", outputs)
//check if document isn't supported by SSM -> update the DocumentLevel status message & send reply accordingly
ssmDocName := parsedMessage.DocumentName
if IsDocumentNotSupportedBySsmAgent(ssmDocName) {
log.Infof("%s is not yet supported by aws-ssm-agent, setting up Document level response accordingly", ssmDocName)
payloadDoc.DocumentTraceOutput = fmt.Sprintf("%s document is not yet supported by amazon-ssm-agent.", ssmDocName)
p.sendDocLevelResponse(*msg.MessageId, contracts.ResultStatusFailed, payloadDoc.DocumentTraceOutput)
}
//update documentInfo in interim cmd state file
documentInfo := commandStateHelper.GetDocumentInfo(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent)
// set document level information which wasn't set previously
documentInfo.AdditionalInfo = payloadDoc.AdditionalInfo
documentInfo.DocumentStatus = payloadDoc.DocumentStatus
documentInfo.DocumentTraceOutput = payloadDoc.DocumentTraceOutput
documentInfo.RuntimeStatus = payloadDoc.RuntimeStatus
//persist final documentInfo.
commandStateHelper.PersistDocumentInfo(log,
documentInfo,
commandID,
*msg.Destination,
appconfig.DefaultLocationOfCurrent)
// Skip sending response when the document requires a reboot
if documentInfo.DocumentStatus == contracts.ResultStatusSuccessAndReboot {
log.Debug("skipping sending response of %v since the document requires a reboot", *msg.MessageId)
return
}
log.Debug("Sending reply on message completion ", outputs)
sendResponse(*msg.MessageId, "", outputs)
//persist : commands execution in completed folder (terminal state folder)
log.Debugf("execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId)
commandStateHelper.MoveCommandState(log,
commandID,
*msg.Destination,
appconfig.DefaultLocationOfCurrent,
appconfig.DefaultLocationOfCompleted)
log.Debugf("Deleting message")
isUpdate := false
for pluginName := range pluginConfigurations {
if pluginName == appconfig.PluginNameAwsAgentUpdate {
isUpdate = true
}
}
if !isUpdate {
err = mdsService.DeleteMessage(log, *msg.MessageId)
if err != nil {
sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
}
} else {
log.Debug("MessageDeletion skipped as it will be handled by external process")
}
}
// NewProcessor initializes a new mds processor with the given parameters.
func NewProcessor(context context.T) *Processor {
messageContext := context.With("[" + name + "]")
log := messageContext.Log()
config := messageContext.AppConfig()
instanceID, err := platform.InstanceID()
if instanceID == "" {
log.Errorf("no instanceID provided, %v", err)
return nil
}
mdsService := newMdsService(config)
agentInfo := contracts.AgentInfo{
Lang: config.Os.Lang,
Name: config.Agent.Name,
Version: config.Agent.Version,
Os: config.Os.Name,
OsVersion: config.Os.Version,
}
agentConfig := contracts.AgentConfiguration{
AgentInfo: agentInfo,
InstanceID: instanceID,
}
// sendCommand and cancelCommand will be processed by separate worker pools
// so we can define the number of workers per each
cancelWaitDuration := 10000 * time.Millisecond
clock := times.DefaultClock
sendCommandTaskPool := task.NewPool(log, config.Mds.CommandWorkersLimit, cancelWaitDuration, clock)
cancelCommandTaskPool := task.NewPool(log, CancelWorkersLimit, cancelWaitDuration, clock)
// create new message processor
orchestrationRootDir := path.Join(appconfig.DefaultDataStorePath, instanceID, appconfig.DefaultCommandRootDirName, config.Agent.OrchestrationRootDir)
replyBuilder := func(pluginID string, results map[string]*contracts.PluginResult) messageContracts.SendReplyPayload {
runtimeStatuses := parser.PrepareRuntimeStatuses(log, results)
return parser.PrepareReplyPayload(pluginID, runtimeStatuses, clock.Now(), agentConfig.AgentInfo)
}
statusReplyBuilder := func(agentInfo contracts.AgentInfo, resultStatus contracts.ResultStatus, documentTraceOutput string) messageContracts.SendReplyPayload {
return parser.PrepareReplyPayloadToUpdateDocumentStatus(agentInfo, resultStatus, documentTraceOutput)
}
// create a stop policy where we will stop after 10 consecutive errors and if time period expires.
processorStopPolicy := newStopPolicy()
// SendResponse is used to send response on plugin completion.
// If pluginID is empty it will send responses of all plugins.
// If pluginID is specified, response will be sent of that particular plugin.
sendResponse := func(messageID string, pluginID string, results map[string]*contracts.PluginResult) {
payloadDoc := replyBuilder(pluginID, results)
processSendReply(log, messageID, mdsService, payloadDoc, processorStopPolicy)
}
// SendDocLevelResponse is used to send document level update
// Specify a new status of the document
sendDocLevelResponse := func(messageID string, resultStatus contracts.ResultStatus, documentTraceOutput string) {
payloadDoc := statusReplyBuilder(agentInfo, resultStatus, documentTraceOutput)
processSendReply(log, messageID, mdsService, payloadDoc, processorStopPolicy)
}
// PersistData is used to persist the data into a bookkeeping folder
persistData := func(msg *ssmmds.Message, bookkeeping string) {
commandStateHelper.PersistData(log, getCommandID(*msg.MessageId), *msg.Destination, bookkeeping, *msg)
}
return &Processor{
context: messageContext,
stopSignal: make(chan bool),
config: agentConfig,
service: mdsService,
pluginRunner: pluginRunner,
sendCommandPool: sendCommandTaskPool,
cancelCommandPool: cancelCommandTaskPool,
buildReply: replyBuilder,
sendResponse: sendResponse,
sendDocLevelResponse: sendDocLevelResponse,
orchestrationRootDir: orchestrationRootDir,
persistData: persistData,
processorStopPolicy: processorStopPolicy,
}
}
