// processCancelCommandMessage processes a single send command message received from MDS. func (p *Processor) processCancelCommandMessage(context context.T, mdsService service.Service, sendCommandPool task.Pool, msg ssmmds.Message) { log := context.Log() log.Debug("Processing cancel command message ", jsonutil.Indent(*msg.Payload)) var parsedMessage messageContracts.CancelPayload err := json.Unmarshal([]byte(*msg.Payload), &parsedMessage) if err != nil { log.Error("format of received cancel message is invalid ", err) err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } return } log.Debugf("ParsedMessage is %v", parsedMessage) //persist in current folder here cancelCmd := initializeCancelCommandState(msg, parsedMessage) commandID := getCommandID(*msg.MessageId) // persist new interim command state in current folder commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, cancelCmd) //remove from pending folder commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending) log.Debugf("Canceling job with id %v...", parsedMessage.CancelMessageID) if found := sendCommandPool.Cancel(parsedMessage.CancelMessageID); !found { log.Debugf("Job with id %v not found (possibly completed)", parsedMessage.CancelMessageID) cancelCmd.DebugInfo = fmt.Sprintf("Command %v couldn't be cancelled", cancelCmd.CancelCommandID) cancelCmd.Status = contracts.ResultStatusFailed } else { cancelCmd.DebugInfo = fmt.Sprintf("Command %v cancelled", cancelCmd.CancelCommandID) cancelCmd.Status = contracts.ResultStatusSuccess } //persist the final status of cancel-message in current folder commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, cancelCmd) //persist : commands execution in completed folder (terminal state folder) log.Debugf("Execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId) commandStateHelper.MoveCommandState(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, appconfig.DefaultLocationOfCompleted) log.Debugf("Deleting message") err = mdsService.DeleteMessage(log, *msg.MessageId) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } }
// processOlderMessagesFromCurrent processes older messages that were persisted in CURRENT folder func (p *Processor) processMessagesFromCurrent(instanceID string) { log := p.context.Log() config := p.context.AppConfig() unprocessedMsgsLocation := path.Join(appconfig.DefaultDataStorePath, instanceID, appconfig.DefaultCommandRootDirName, appconfig.DefaultLocationOfState, appconfig.DefaultLocationOfCurrent) if isDirectoryEmpty, _ := fileutil.IsDirEmpty(unprocessedMsgsLocation); isDirectoryEmpty { log.Debugf("no older messages to process from %v", unprocessedMsgsLocation) } else { //get all older messages from previous run of agent files, err := ioutil.ReadDir(unprocessedMsgsLocation) //TODO: revisit this when bookkeeping is made invasive if err != nil { log.Errorf("skipping reading inprogress messages from %v. unexpected error encountered - %v", unprocessedMsgsLocation, err) return } //iterate through all old executing messages for _, f := range files { log.Debugf("processing previously unexecuted message - %v", f.Name()) //construct the absolute path - safely assuming that interim state for older messages are already present in Current folder file := path.Join(appconfig.DefaultDataStorePath, instanceID, appconfig.DefaultCommandRootDirName, appconfig.DefaultLocationOfState, appconfig.DefaultLocationOfCurrent, f.Name()) var oldCmdState messageContracts.CommandState //parse the message //TODO: Not all messages in Current folder correspond to SendCommand. if err := jsonutil.UnmarshalFile(file, &oldCmdState); err != nil { log.Errorf("skipping processsing of previously unexecuted messages. encountered error %v while reading unprocessed message from file - %v", err, f) } else { if oldCmdState.DocumentInformation.RunCount >= config.Mds.CommandRetryLimit { //TODO: Move command to corrupt/failed // do not process as the command has failed too many times break } pluginOutputs := make(map[string]*contracts.PluginResult) // increment the command run count oldCmdState.DocumentInformation.RunCount++ // Update reboot status for v := range oldCmdState.PluginsInformation { plugin := oldCmdState.PluginsInformation[v] if plugin.HasExecuted && plugin.Result.Status == contracts.ResultStatusSuccessAndReboot { log.Debugf("plugin %v has completed a reboot. Setting status to Success.", v) plugin.Result.Status = contracts.ResultStatusSuccess oldCmdState.PluginsInformation[v] = plugin pluginOutputs[v] = &plugin.Result p.sendResponse(oldCmdState.DocumentInformation.MessageID, v, pluginOutputs) } } // func PersistData(log log.T, commandID, instanceID, locationFolder string, object interface{}) { commandStateHelper.PersistData(log, oldCmdState.DocumentInformation.CommandID, instanceID, appconfig.DefaultLocationOfCurrent, oldCmdState) //Submit the work to Job Pool so that we don't block for processing of new messages err := p.sendCommandPool.Submit(log, oldCmdState.DocumentInformation.MessageID, func(cancelFlag task.CancelFlag) { p.runCmdsUsingCmdState(p.context.With("[messageID="+oldCmdState.DocumentInformation.MessageID+"]"), p.service, p.pluginRunner, cancelFlag, p.buildReply, p.sendResponse, oldCmdState) }) if err != nil { log.Error("SendCommand failed for previously unexecuted commands", err) return } } } } }
// processSendCommandMessage processes a single send command message received from MDS. func (p *Processor) processSendCommandMessage(context context.T, mdsService service.Service, messagesOrchestrationRootDir string, runPlugins PluginRunner, cancelFlag task.CancelFlag, buildReply replyBuilder, sendResponse engine.SendResponse, msg ssmmds.Message) { commandID := getCommandID(*msg.MessageId) log := context.Log() log.Debug("Processing send command message ", *msg.MessageId) log.Trace("Processing send command message ", jsonutil.Indent(*msg.Payload)) parsedMessage, err := parser.ParseMessageWithParams(log, *msg.Payload) if err != nil { log.Error("format of received message is invalid ", err) err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } return } parsedMessageContent, _ := jsonutil.Marshal(parsedMessage) log.Debug("ParsedMessage is ", jsonutil.Indent(parsedMessageContent)) // adapt plugin configuration format from MDS to plugin expected format s3KeyPrefix := path.Join(parsedMessage.OutputS3KeyPrefix, parsedMessage.CommandID, *msg.Destination) messageOrchestrationDirectory := filepath.Join(messagesOrchestrationRootDir, commandID) // Check if it is a managed instance and its executing managed instance incompatible AWS SSM public document. // A few public AWS SSM documents contain code which is not compatible when run on managed instances. // isManagedInstanceIncompatibleAWSSSMDocument makes sure to find such documents at runtime and replace the incompatible code. isMI, err := isManagedInstance() if err != nil { log.Errorf("Error determining managed instance. error: %v", err) } if isMI && isManagedInstanceIncompatibleAWSSSMDocument(parsedMessage.DocumentName) { log.Debugf("Running Incompatible AWS SSM Document %v on managed instance", parsedMessage.DocumentName) if parsedMessage, err = removeDependencyOnInstanceMetadataForManagedInstance(context, parsedMessage); err != nil { return } } pluginConfigurations := getPluginConfigurations( parsedMessage.DocumentContent.RuntimeConfig, messageOrchestrationDirectory, parsedMessage.OutputS3BucketName, s3KeyPrefix, *msg.MessageId) //persist : all information in current folder log.Info("Persisting message in current execution folder") //Data format persisted in Current Folder is defined by the struct - CommandState interimCmdState := initializeCommandState(pluginConfigurations, msg, parsedMessage) // persist new interim command state in current folder commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, interimCmdState) //Deleting from pending folder since the command is getting executed commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending) log.Debug("Running plugins...") outputs := runPlugins(context, *msg.MessageId, pluginConfigurations, sendResponse, cancelFlag) pluginOutputContent, _ := jsonutil.Marshal(outputs) log.Debugf("Plugin outputs %v", jsonutil.Indent(pluginOutputContent)) payloadDoc := buildReply("", outputs) //check if document isn't supported by SSM -> update the DocumentLevel status message & send reply accordingly ssmDocName := parsedMessage.DocumentName if IsDocumentNotSupportedBySsmAgent(ssmDocName) { log.Infof("%s is not yet supported by aws-ssm-agent, setting up Document level response accordingly", ssmDocName) payloadDoc.DocumentTraceOutput = fmt.Sprintf("%s document is not yet supported by amazon-ssm-agent.", ssmDocName) p.sendDocLevelResponse(*msg.MessageId, contracts.ResultStatusFailed, payloadDoc.DocumentTraceOutput) } //update documentInfo in interim cmd state file documentInfo := commandStateHelper.GetDocumentInfo(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent) // set document level information which wasn't set previously documentInfo.AdditionalInfo = payloadDoc.AdditionalInfo documentInfo.DocumentStatus = payloadDoc.DocumentStatus documentInfo.DocumentTraceOutput = payloadDoc.DocumentTraceOutput documentInfo.RuntimeStatus = payloadDoc.RuntimeStatus //persist final documentInfo. commandStateHelper.PersistDocumentInfo(log, documentInfo, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent) // Skip sending response when the document requires a reboot if documentInfo.DocumentStatus == contracts.ResultStatusSuccessAndReboot { log.Debug("skipping sending response of %v since the document requires a reboot", *msg.MessageId) return } log.Debug("Sending reply on message completion ", outputs) sendResponse(*msg.MessageId, "", outputs) //persist : commands execution in completed folder (terminal state folder) log.Debugf("execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId) commandStateHelper.MoveCommandState(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, appconfig.DefaultLocationOfCompleted) log.Debugf("Deleting message") isUpdate := false for pluginName := range pluginConfigurations { if pluginName == appconfig.PluginNameAwsAgentUpdate { isUpdate = true } } if !isUpdate { err = mdsService.DeleteMessage(log, *msg.MessageId) if err != nil { sdkutil.HandleAwsError(log, err, p.processorStopPolicy) } } else { log.Debug("MessageDeletion skipped as it will be handled by external process") } }
// NewProcessor initializes a new mds processor with the given parameters. func NewProcessor(context context.T) *Processor { messageContext := context.With("[" + name + "]") log := messageContext.Log() config := messageContext.AppConfig() instanceID, err := platform.InstanceID() if instanceID == "" { log.Errorf("no instanceID provided, %v", err) return nil } mdsService := newMdsService(config) agentInfo := contracts.AgentInfo{ Lang: config.Os.Lang, Name: config.Agent.Name, Version: config.Agent.Version, Os: config.Os.Name, OsVersion: config.Os.Version, } agentConfig := contracts.AgentConfiguration{ AgentInfo: agentInfo, InstanceID: instanceID, } // sendCommand and cancelCommand will be processed by separate worker pools // so we can define the number of workers per each cancelWaitDuration := 10000 * time.Millisecond clock := times.DefaultClock sendCommandTaskPool := task.NewPool(log, config.Mds.CommandWorkersLimit, cancelWaitDuration, clock) cancelCommandTaskPool := task.NewPool(log, CancelWorkersLimit, cancelWaitDuration, clock) // create new message processor orchestrationRootDir := path.Join(appconfig.DefaultDataStorePath, instanceID, appconfig.DefaultCommandRootDirName, config.Agent.OrchestrationRootDir) replyBuilder := func(pluginID string, results map[string]*contracts.PluginResult) messageContracts.SendReplyPayload { runtimeStatuses := parser.PrepareRuntimeStatuses(log, results) return parser.PrepareReplyPayload(pluginID, runtimeStatuses, clock.Now(), agentConfig.AgentInfo) } statusReplyBuilder := func(agentInfo contracts.AgentInfo, resultStatus contracts.ResultStatus, documentTraceOutput string) messageContracts.SendReplyPayload { return parser.PrepareReplyPayloadToUpdateDocumentStatus(agentInfo, resultStatus, documentTraceOutput) } // create a stop policy where we will stop after 10 consecutive errors and if time period expires. processorStopPolicy := newStopPolicy() // SendResponse is used to send response on plugin completion. // If pluginID is empty it will send responses of all plugins. // If pluginID is specified, response will be sent of that particular plugin. sendResponse := func(messageID string, pluginID string, results map[string]*contracts.PluginResult) { payloadDoc := replyBuilder(pluginID, results) processSendReply(log, messageID, mdsService, payloadDoc, processorStopPolicy) } // SendDocLevelResponse is used to send document level update // Specify a new status of the document sendDocLevelResponse := func(messageID string, resultStatus contracts.ResultStatus, documentTraceOutput string) { payloadDoc := statusReplyBuilder(agentInfo, resultStatus, documentTraceOutput) processSendReply(log, messageID, mdsService, payloadDoc, processorStopPolicy) } // PersistData is used to persist the data into a bookkeeping folder persistData := func(msg *ssmmds.Message, bookkeeping string) { commandStateHelper.PersistData(log, getCommandID(*msg.MessageId), *msg.Destination, bookkeeping, *msg) } return &Processor{ context: messageContext, stopSignal: make(chan bool), config: agentConfig, service: mdsService, pluginRunner: pluginRunner, sendCommandPool: sendCommandTaskPool, cancelCommandPool: cancelCommandTaskPool, buildReply: replyBuilder, sendResponse: sendResponse, sendDocLevelResponse: sendDocLevelResponse, orchestrationRootDir: orchestrationRootDir, persistData: persistData, processorStopPolicy: processorStopPolicy, } }