Example #1
0
// processCancelCommandMessage processes a single send command message received from MDS.
func (p *Processor) processCancelCommandMessage(context context.T,
	mdsService service.Service,
	sendCommandPool task.Pool,
	msg ssmmds.Message) {

	log := context.Log()
	log.Debug("Processing cancel command message ", jsonutil.Indent(*msg.Payload))

	var parsedMessage messageContracts.CancelPayload
	err := json.Unmarshal([]byte(*msg.Payload), &parsedMessage)
	if err != nil {
		log.Error("format of received cancel message is invalid ", err)
		err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException)
		if err != nil {
			sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
		}
		return
	}
	log.Debugf("ParsedMessage is %v", parsedMessage)

	//persist in current folder here
	cancelCmd := initializeCancelCommandState(msg, parsedMessage)
	commandID := getCommandID(*msg.MessageId)
	// persist new interim command state in current folder
	commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, cancelCmd)

	//remove from pending folder
	commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending)

	log.Debugf("Canceling job with id %v...", parsedMessage.CancelMessageID)

	if found := sendCommandPool.Cancel(parsedMessage.CancelMessageID); !found {
		log.Debugf("Job with id %v not found (possibly completed)", parsedMessage.CancelMessageID)
		cancelCmd.DebugInfo = fmt.Sprintf("Command %v couldn't be cancelled", cancelCmd.CancelCommandID)
		cancelCmd.Status = contracts.ResultStatusFailed
	} else {
		cancelCmd.DebugInfo = fmt.Sprintf("Command %v cancelled", cancelCmd.CancelCommandID)
		cancelCmd.Status = contracts.ResultStatusSuccess
	}

	//persist the final status of cancel-message in current folder
	commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, cancelCmd)

	//persist : commands execution in completed folder (terminal state folder)
	log.Debugf("Execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId)

	commandStateHelper.MoveCommandState(log,
		commandID,
		*msg.Destination,
		appconfig.DefaultLocationOfCurrent,
		appconfig.DefaultLocationOfCompleted)

	log.Debugf("Deleting message")
	err = mdsService.DeleteMessage(log, *msg.MessageId)
	if err != nil {
		sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
	}
}
Example #2
0
// processOlderMessagesFromCurrent processes older messages that were persisted in CURRENT folder
func (p *Processor) processMessagesFromCurrent(instanceID string) {
	log := p.context.Log()
	config := p.context.AppConfig()

	unprocessedMsgsLocation := path.Join(appconfig.DefaultDataStorePath,
		instanceID,
		appconfig.DefaultCommandRootDirName,
		appconfig.DefaultLocationOfState,
		appconfig.DefaultLocationOfCurrent)

	if isDirectoryEmpty, _ := fileutil.IsDirEmpty(unprocessedMsgsLocation); isDirectoryEmpty {

		log.Debugf("no older messages to process from %v", unprocessedMsgsLocation)

	} else {

		//get all older messages from previous run of agent
		files, err := ioutil.ReadDir(unprocessedMsgsLocation)

		//TODO:  revisit this when bookkeeping is made invasive
		if err != nil {
			log.Errorf("skipping reading inprogress messages from %v. unexpected error encountered - %v", unprocessedMsgsLocation, err)
			return
		}

		//iterate through all old executing messages
		for _, f := range files {
			log.Debugf("processing previously unexecuted message - %v", f.Name())

			//construct the absolute path - safely assuming that interim state for older messages are already present in Current folder
			file := path.Join(appconfig.DefaultDataStorePath,
				instanceID,
				appconfig.DefaultCommandRootDirName,
				appconfig.DefaultLocationOfState,
				appconfig.DefaultLocationOfCurrent,
				f.Name())

			var oldCmdState messageContracts.CommandState

			//parse the message
			//TODO:  Not all messages in Current folder correspond to SendCommand.
			if err := jsonutil.UnmarshalFile(file, &oldCmdState); err != nil {
				log.Errorf("skipping processsing of previously unexecuted messages. encountered error %v while reading unprocessed message from file - %v", err, f)
			} else {
				if oldCmdState.DocumentInformation.RunCount >= config.Mds.CommandRetryLimit {
					//TODO:  Move command to corrupt/failed
					// do not process as the command has failed too many times
					break
				}

				pluginOutputs := make(map[string]*contracts.PluginResult)

				// increment the command run count
				oldCmdState.DocumentInformation.RunCount++
				// Update reboot status
				for v := range oldCmdState.PluginsInformation {
					plugin := oldCmdState.PluginsInformation[v]
					if plugin.HasExecuted && plugin.Result.Status == contracts.ResultStatusSuccessAndReboot {
						log.Debugf("plugin %v has completed a reboot. Setting status to Success.", v)
						plugin.Result.Status = contracts.ResultStatusSuccess
						oldCmdState.PluginsInformation[v] = plugin
						pluginOutputs[v] = &plugin.Result
						p.sendResponse(oldCmdState.DocumentInformation.MessageID, v, pluginOutputs)
					}
				}

				// func PersistData(log log.T, commandID, instanceID, locationFolder string, object interface{}) {
				commandStateHelper.PersistData(log, oldCmdState.DocumentInformation.CommandID, instanceID, appconfig.DefaultLocationOfCurrent, oldCmdState)

				//Submit the work to Job Pool so that we don't block for processing of new messages
				err := p.sendCommandPool.Submit(log, oldCmdState.DocumentInformation.MessageID, func(cancelFlag task.CancelFlag) {
					p.runCmdsUsingCmdState(p.context.With("[messageID="+oldCmdState.DocumentInformation.MessageID+"]"),
						p.service,
						p.pluginRunner,
						cancelFlag,
						p.buildReply,
						p.sendResponse,
						oldCmdState)
				})
				if err != nil {
					log.Error("SendCommand failed for previously unexecuted commands", err)
					return
				}
			}
		}
	}
}
Example #3
0
// processSendCommandMessage processes a single send command message received from MDS.
func (p *Processor) processSendCommandMessage(context context.T,
	mdsService service.Service,
	messagesOrchestrationRootDir string,
	runPlugins PluginRunner,
	cancelFlag task.CancelFlag,
	buildReply replyBuilder,
	sendResponse engine.SendResponse,
	msg ssmmds.Message) {

	commandID := getCommandID(*msg.MessageId)

	log := context.Log()
	log.Debug("Processing send command message ", *msg.MessageId)
	log.Trace("Processing send command message ", jsonutil.Indent(*msg.Payload))

	parsedMessage, err := parser.ParseMessageWithParams(log, *msg.Payload)
	if err != nil {
		log.Error("format of received message is invalid ", err)
		err = mdsService.FailMessage(log, *msg.MessageId, service.InternalHandlerException)
		if err != nil {
			sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
		}
		return
	}

	parsedMessageContent, _ := jsonutil.Marshal(parsedMessage)
	log.Debug("ParsedMessage is ", jsonutil.Indent(parsedMessageContent))

	// adapt plugin configuration format from MDS to plugin expected format
	s3KeyPrefix := path.Join(parsedMessage.OutputS3KeyPrefix, parsedMessage.CommandID, *msg.Destination)

	messageOrchestrationDirectory := filepath.Join(messagesOrchestrationRootDir, commandID)

	// Check if it is a managed instance and its executing managed instance incompatible AWS SSM public document.
	// A few public AWS SSM documents contain code which is not compatible when run on managed instances.
	// isManagedInstanceIncompatibleAWSSSMDocument makes sure to find such documents at runtime and replace the incompatible code.
	isMI, err := isManagedInstance()
	if err != nil {
		log.Errorf("Error determining managed instance. error: %v", err)
	}

	if isMI && isManagedInstanceIncompatibleAWSSSMDocument(parsedMessage.DocumentName) {
		log.Debugf("Running Incompatible AWS SSM Document %v on managed instance", parsedMessage.DocumentName)
		if parsedMessage, err = removeDependencyOnInstanceMetadataForManagedInstance(context, parsedMessage); err != nil {
			return
		}
	}

	pluginConfigurations := getPluginConfigurations(
		parsedMessage.DocumentContent.RuntimeConfig,
		messageOrchestrationDirectory,
		parsedMessage.OutputS3BucketName,
		s3KeyPrefix,
		*msg.MessageId)

	//persist : all information in current folder
	log.Info("Persisting message in current execution folder")

	//Data format persisted in Current Folder is defined by the struct - CommandState
	interimCmdState := initializeCommandState(pluginConfigurations, msg, parsedMessage)

	// persist new interim command state in current folder
	commandStateHelper.PersistData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent, interimCmdState)

	//Deleting from pending folder since the command is getting executed
	commandStateHelper.RemoveData(log, commandID, *msg.Destination, appconfig.DefaultLocationOfPending)

	log.Debug("Running plugins...")
	outputs := runPlugins(context, *msg.MessageId, pluginConfigurations, sendResponse, cancelFlag)
	pluginOutputContent, _ := jsonutil.Marshal(outputs)
	log.Debugf("Plugin outputs %v", jsonutil.Indent(pluginOutputContent))

	payloadDoc := buildReply("", outputs)

	//check if document isn't supported by SSM -> update the DocumentLevel status message & send reply accordingly
	ssmDocName := parsedMessage.DocumentName
	if IsDocumentNotSupportedBySsmAgent(ssmDocName) {
		log.Infof("%s is not yet supported by aws-ssm-agent, setting up Document level response accordingly", ssmDocName)
		payloadDoc.DocumentTraceOutput = fmt.Sprintf("%s document is not yet supported by amazon-ssm-agent.", ssmDocName)
		p.sendDocLevelResponse(*msg.MessageId, contracts.ResultStatusFailed, payloadDoc.DocumentTraceOutput)
	}

	//update documentInfo in interim cmd state file
	documentInfo := commandStateHelper.GetDocumentInfo(log, commandID, *msg.Destination, appconfig.DefaultLocationOfCurrent)

	// set document level information which wasn't set previously
	documentInfo.AdditionalInfo = payloadDoc.AdditionalInfo
	documentInfo.DocumentStatus = payloadDoc.DocumentStatus
	documentInfo.DocumentTraceOutput = payloadDoc.DocumentTraceOutput
	documentInfo.RuntimeStatus = payloadDoc.RuntimeStatus

	//persist final documentInfo.
	commandStateHelper.PersistDocumentInfo(log,
		documentInfo,
		commandID,
		*msg.Destination,
		appconfig.DefaultLocationOfCurrent)

	// Skip sending response when the document requires a reboot
	if documentInfo.DocumentStatus == contracts.ResultStatusSuccessAndReboot {
		log.Debug("skipping sending response of %v since the document requires a reboot", *msg.MessageId)
		return
	}

	log.Debug("Sending reply on message completion ", outputs)
	sendResponse(*msg.MessageId, "", outputs)

	//persist : commands execution in completed folder (terminal state folder)
	log.Debugf("execution of %v is over. Moving interimState file from Current to Completed folder", *msg.MessageId)

	commandStateHelper.MoveCommandState(log,
		commandID,
		*msg.Destination,
		appconfig.DefaultLocationOfCurrent,
		appconfig.DefaultLocationOfCompleted)

	log.Debugf("Deleting message")
	isUpdate := false
	for pluginName := range pluginConfigurations {
		if pluginName == appconfig.PluginNameAwsAgentUpdate {
			isUpdate = true
		}
	}
	if !isUpdate {
		err = mdsService.DeleteMessage(log, *msg.MessageId)
		if err != nil {
			sdkutil.HandleAwsError(log, err, p.processorStopPolicy)
		}
	} else {
		log.Debug("MessageDeletion skipped as it will be handled by external process")
	}
}
Example #4
0
// NewProcessor initializes a new mds processor with the given parameters.
func NewProcessor(context context.T) *Processor {
	messageContext := context.With("[" + name + "]")
	log := messageContext.Log()
	config := messageContext.AppConfig()

	instanceID, err := platform.InstanceID()
	if instanceID == "" {
		log.Errorf("no instanceID provided, %v", err)
		return nil
	}

	mdsService := newMdsService(config)

	agentInfo := contracts.AgentInfo{
		Lang:      config.Os.Lang,
		Name:      config.Agent.Name,
		Version:   config.Agent.Version,
		Os:        config.Os.Name,
		OsVersion: config.Os.Version,
	}

	agentConfig := contracts.AgentConfiguration{
		AgentInfo:  agentInfo,
		InstanceID: instanceID,
	}

	// sendCommand and cancelCommand will be processed by separate worker pools
	// so we can define the number of workers per each
	cancelWaitDuration := 10000 * time.Millisecond
	clock := times.DefaultClock
	sendCommandTaskPool := task.NewPool(log, config.Mds.CommandWorkersLimit, cancelWaitDuration, clock)
	cancelCommandTaskPool := task.NewPool(log, CancelWorkersLimit, cancelWaitDuration, clock)

	// create new message processor
	orchestrationRootDir := path.Join(appconfig.DefaultDataStorePath, instanceID, appconfig.DefaultCommandRootDirName, config.Agent.OrchestrationRootDir)

	replyBuilder := func(pluginID string, results map[string]*contracts.PluginResult) messageContracts.SendReplyPayload {
		runtimeStatuses := parser.PrepareRuntimeStatuses(log, results)
		return parser.PrepareReplyPayload(pluginID, runtimeStatuses, clock.Now(), agentConfig.AgentInfo)
	}

	statusReplyBuilder := func(agentInfo contracts.AgentInfo, resultStatus contracts.ResultStatus, documentTraceOutput string) messageContracts.SendReplyPayload {
		return parser.PrepareReplyPayloadToUpdateDocumentStatus(agentInfo, resultStatus, documentTraceOutput)

	}
	// create a stop policy where we will stop after 10 consecutive errors and if time period expires.
	processorStopPolicy := newStopPolicy()

	// SendResponse is used to send response on plugin completion.
	// If pluginID is empty it will send responses of all plugins.
	// If pluginID is specified, response will be sent of that particular plugin.
	sendResponse := func(messageID string, pluginID string, results map[string]*contracts.PluginResult) {
		payloadDoc := replyBuilder(pluginID, results)
		processSendReply(log, messageID, mdsService, payloadDoc, processorStopPolicy)
	}

	// SendDocLevelResponse is used to send document level update
	// Specify a new status of the document
	sendDocLevelResponse := func(messageID string, resultStatus contracts.ResultStatus, documentTraceOutput string) {
		payloadDoc := statusReplyBuilder(agentInfo, resultStatus, documentTraceOutput)
		processSendReply(log, messageID, mdsService, payloadDoc, processorStopPolicy)
	}

	// PersistData is used to persist the data into a bookkeeping folder
	persistData := func(msg *ssmmds.Message, bookkeeping string) {
		commandStateHelper.PersistData(log, getCommandID(*msg.MessageId), *msg.Destination, bookkeeping, *msg)
	}

	return &Processor{
		context:              messageContext,
		stopSignal:           make(chan bool),
		config:               agentConfig,
		service:              mdsService,
		pluginRunner:         pluginRunner,
		sendCommandPool:      sendCommandTaskPool,
		cancelCommandPool:    cancelCommandTaskPool,
		buildReply:           replyBuilder,
		sendResponse:         sendResponse,
		sendDocLevelResponse: sendDocLevelResponse,
		orchestrationRootDir: orchestrationRootDir,
		persistData:          persistData,
		processorStopPolicy:  processorStopPolicy,
	}
}