Example #1
0
// process a message from the SQS queue. This should be run inside a goroutine.
func processMessage(q *sqs.Queue, m sqs.Message, wo work_order.WorkOrder, wg *sync.WaitGroup) {
	logger.Debug("Starting process on %d from '%s'", wo.Id, m.MessageId)

	// start heartbeat
	beat := heartbeat.Start(q, &m)

	// execute the work
	err := wo.Execute()
	if err != nil {
		logger.Error("Error executing: %d - %v", wo.Id, err)
	}

	// send response back to devops-web
	wo.Report()

	// stop the heartbeat
	beat.Stop()

	// delete message
	logger.Debug("Deleting message: %s", m.MessageId)
	_, err = q.DeleteMessage(&m)
	if err != nil {
		logger.Error("ERROR: Couldn't delete message: %s - %v", m.MessageId, err)
	}

	// exit this goroutine
	wg.Done()
}
Example #2
0
// Execute a WorkOrder and populate its response object
func (wo *WorkOrder) Execute() (error error) {
	logger.Info("Starting work on WorkOrder: %d with \"%s\"", wo.Id, wo.Message)

	// setup command to be run with arguments from the command line
	wo_args := strings.Split(wo.Message, " ")
	base_args := strings.Split(os.Getenv("CMD_BASE"), " ")
	cmd := exec.Command(base_args[0])
	cmd.Args = append(base_args[0:], wo_args[0:]...)
	cmd.Dir = os.Getenv("CMD_DIR")

	// collect stdout and stderr
	var output bytes.Buffer
	cmd.Stdout = &output
	cmd.Stderr = &output

	// start timing command
	start_time := time.Now()

	// execute the command
	if err := cmd.Start(); err != nil {
		logger.Error("cmd.Start:", err)
		error = err
	}

	// http://stackoverflow.com/questions/10385551/get-exit-code-go
	if err := cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0

			// This works on both Unix and Windows. Although package
			// syscall is generally platform dependent, WaitStatus is
			// defined for both Unix and Windows and in both cases has
			// an ExitStatus() method with the same signature.
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				// logger.Info("Exit Status: %d", status.ExitStatus())
				wo.response.Result.ExitStatus = status.ExitStatus()
			}
		} else {
			logger.Error("cmd.Wait: %v", err)
			error = err
		}
	}

	// calculate the time taken to complete the command
	end_time := time.Now()
	wo.response.TimeTaken = end_time.Sub(start_time).Seconds()
	wo.response.CompletedAt = &end_time

	// attach the output of the command to the result message
	wo.response.Result.Message = output.String()

	logger.Info("Completed WorkOrder: %d", wo.Id)
	return
}
Example #3
0
// Report on the result of the WorkOrders execution.
// This method requires that the WorkOrder has been Executed.
func (wo *WorkOrder) Report() (error error) {
	report_queue := os.Getenv("SQS_REPORT_QUEUE")
	logger.Info("Sending response to '%s' for: %d", report_queue, wo.Id)

	// prepare the response object
	wo.response.Id = wo.Id

	// create sqs client
	client, err := sqs.NewFrom(os.Getenv("SQS_WORKER_ACCESS_KEY"), os.Getenv("SQS_WORKER_SECRET_KEY"), "us-east-1")
	if err != nil {
		logger.Error("Could not report: %d - %v", wo.Id, err)
		error = err
		return
	}

	// get the SQS queue
	queue, err := client.GetQueue(report_queue)
	if err != nil {
		logger.Error("REPORT QUEUE ERROR: %d - %v", wo.Id, err)
		error = err
		return
	}

	// trim the message so that it will fit in SQS
	if len(wo.response.Result.Message) > 60000 {
		logger.Info("Trimming WorkOrder %d message down from %d chars", wo.Id, len(wo.response.Result.Message))
		wo.response.Result.Message = wo.response.Result.Message[0:30000] + "\n\n...(truncated)...\n\n" + wo.response.Result.Message[len(wo.response.Result.Message)-30000:]
	}

	// marshal the response object into json
	data, err := json.Marshal(wo.response)
	if err != nil {
		logger.Error("Could not convert response to JSON for: %d - %v", wo.Id, err)
		error = err
		return
	}

	// send the report to the queue
	_, err = queue.SendMessage(string(data))
	if err != nil {
		logger.Error("Could not report: %d - %v", wo.Id, err)
		error = err
	}

	return
}
Example #4
0
// Send a heartbeat to SQS notifying it that we are still working on the message.
func (heartbeat *Heartbeat) beat(t time.Time) {
	logger.Debug("Sending heartbeat for: %s", heartbeat.Message.MessageId)

	// change the sqs message visibility
	_, err := heartbeat.Queue.ChangeMessageVisibility(heartbeat.Message, 2*60)
	if err != nil {
		logger.Error("HEARTBEAT ERROR: messageId: %s - %v", heartbeat.Message.MessageId, err)
	}
}
Example #5
0
func main() {
	logger.Info("Sending messages...")

	// create sqs client
	client, err := sqs.NewFrom(os.Getenv("ADMIN_AWS_ACCESS_KEY_ID"), os.Getenv("ADMIN_AWS_SECRET_ACCESS_KEY"), "us-east-1")
	if err != nil {
		logger.Fatal("CLIENT ERROR:", err, "asdf", "asdfdasfd")
	}

	// get the SQS queue
	queue, err := client.GetQueue(os.Getenv("SQS_RECIEVE_QUEUE"))
	if err != nil {
		logger.Fatal("QUEUE ERROR:", err)
	}

	for i := 0; i < 100; i++ {
		var wo work_order.WorkOrder
		current_time := time.Now()

		wo.Id = i
		wo.JobId = 1
		wo.Message = strconv.Itoa(rand.Intn(80))
		wo.CreatedAt = &current_time
		wo.UpdatedAt = &current_time
		wo.Queue = os.Getenv("SQS_RECIEVE_QUEUE")

		data, err := json.Marshal(wo)
		if err != nil {
			logger.Error("JSON error: %v", err)
		}
		queue.SendMessage(string(data))
	}

	// quit
	logger.Info("Exiting.")
	os.Exit(0)

}
Example #6
0
func main() {
	logger.Info("Starting worker v%s", VERSION)

	// get worker count
	workers, err := strconv.Atoi(os.Getenv("WORKER_COUNT"))
	if err != nil {
		workers = 10
	}
	logger.Info("Worker count: %d", workers)

	// access key, secret key, receive queue and report queue should be in ENV variables
	logger.Info("SQS queue: %s", os.Getenv("SQS_WORKER_QUEUE"))

	// create sqs client
	client, err := sqs.NewFrom(os.Getenv("SQS_WORKER_ACCESS_KEY"), os.Getenv("SQS_WORKER_SECRET_KEY"), "us-east-1")
	if err != nil {
		logger.Fatal("CLIENT ERROR: %v", err)
	}

	// get the SQS queue
	queue, err := client.GetQueue(os.Getenv("SQS_WORKER_QUEUE"))
	if err != nil {
		logger.Fatal("QUEUE ERROR: %v", err)
	}

	logger.Info("Worker started.")

	// create the wait group
	var wg sync.WaitGroup

	for {
		// get some messages from the sqs queue
		logger.Debug("Checking for messages on the queue...")
		resp, err := queue.ReceiveMessageWithVisibilityTimeout(workers, 60)
		if err != nil {
			logger.Error("Could not receive messages: %v", err)
			time.Sleep(10 * time.Second)
		}

		if cap(resp.Messages) == 0 {
			logger.Debug("Did not find any messages on the queue.")
		}

		// for each message
		for _, message := range resp.Messages {
			// get the message details
			wo, err := work_order.NewFromJson(message.Body)
			if err != nil {
				logger.Error("Could not process SQS message: %s with JSON ERROR: %v", message.MessageId, err)
			} else {
				// process the message in a goroutine
				wg.Add(1)
				go processMessage(queue, message, wo, &wg)
			}
		}

		// wait for each goroutine to exit
		wg.Wait()
	}

}