Ejemplo n.º 1
0
func NewSQSQueue(s *sqs.SQS, name string) (*queue.Queue, error) {
	stackName := Getenv("AWS_STACK_NAME", defaultStackName)
	if stackName != "" {
		stackName += "-"
	}
	return queue.New(s, stackName+name)
}
Ejemplo n.º 2
0
func main() {
	// Create SQS instance
	s := sqs.New(session.New(), &aws.Config{Region: aws.String("us-west-2")})

	// Create Queue instance
	q, err := queue.New(s, "test")
	if err != nil {
		log.Fatal(err)
	}

	for {
		messages, err := q.ReceiveMessage(option.MaxNumberOfMessages(10))
		if err != nil {
			log.Println(err)
			continue
		}

		if len(messages) > 0 {
			for _, m := range messages {
				log.Println(*m.Body)
				q.DeleteMessage(m.ReceiptHandle)
			}
		}

	}
}
Ejemplo n.º 3
0
func TestWorker(t *testing.T) {
	assert := assert.New(t)

	if !*doInteg {
		t.Skip("skipping integration testing")
	}

	queueName := os.Getenv("TEST_SQS_QUEUE_NAME")
	if queueName == "" {
		t.Fatal("TEST_SQS_QUEUE_NAME must be specified")
	}

	relayTo := os.Getenv("TEST_SQS_RELAY_TO")
	if relayTo == "" {
		t.Fatal("TEST_SQS_RELAY_TO must be specified")
	}

	noExistRelayTo := os.Getenv("NOEXIST_TEST_SQS_RELAY_TO")
	if noExistRelayTo == "" {
		t.Fatal("NOEXIST_TEST_SQS_RELAY_TO must be specified")
	}

	drv := &pqDriver{
		workerID: "testing-1",
		db:       newTestDriver(),
	}
	drv.db.Exec("DELETE FROM queue;")
	defer drv.db.Exec("DELETE FROM queue;")

	sqsSvc := sqs.New(nil)
	q, err := queue.New(sqsSvc, queueName)
	if err != nil {
		t.Fatal(err)
	}

	consumer := NewConsumer(drv.workerID, drv, q)
	relay := NewRelay(sqsSvc)

	sender := NewSender(q)
	if !assert.NoError(sender.SendMessage(5, relayTo, "payload-1")) {
		return
	}
	if !assert.NoError(sender.SendMessage(5, noExistRelayTo, "payload-2")) {
		return
	}

	workerConfig := &WorkerConfig{
		ID: drv.workerID,
	}

	w := NewWorker(workerConfig, drv, consumer, relay)

	var numMessages int64
	for i := 0; i < 2; i++ {
		n, err := w.consume()
		if !assert.NoError(err) {
			return
		}
		numMessages += n
	}
	if !assert.Equal(int64(2), numMessages) {
		return
	}

	time.Sleep(6 * time.Second)
	now := time.Now()
	{
		n, err := w.markActive(now)
		assert.NoError(err)
		assert.Equal(int64(2), n)
	}

	{
		err := w.release()
		assert.NoError(err)
	}

	{
		err := w.release()
		assert.NoError(err)
	}
}
Ejemplo n.º 4
0
func (c *BenchCommand) Recv(args []string) int {
	var config BenchRecvConfig

	if err := envconfig.Process("delayd2", &config); err != nil {
		c.Ui.Error(fmt.Sprintf("Unable to get configuration from envvars: %s", err))
		return 1
	}

	cmdFlags := flag.NewFlagSet("batch recv", flag.ContinueOnError)
	cmdFlags.IntVar(&config.NumberOfMessages, "n", 1000, "number of messages")

	if err := cmdFlags.Parse(args); err != nil {
		c.Ui.Error(c.Help())
		return 1
	}

	sqsSvc := sqs.New(session.New())
	q, err := queue.New(sqsSvc, config.QueueName)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Unable to initialize SQS connection: %s", err))
		return 1
	}

	log.Print("bench: --- Configuration ---")
	log.Printf("bench: Number of messages to receive: %d", config.NumberOfMessages)
	log.Printf("bench: DELAYD2_QUEUE_NAME=%s", config.QueueName)

	errCh := make(chan error)

	log.Print("bench: starting")
	go func() {
		payloads := map[string]struct{}{}
		for {
			log.Printf("%d messages processed", len(payloads))
			if len(payloads) >= config.NumberOfMessages {
				break
			}

			messages, err := q.ReceiveMessage(
				option.MaxNumberOfMessages(10),
				option.UseAllAttribute(),
			)
			// continue even if we get an error
			if err != nil {
				log.Printf("unable to receive messages but continuing...: %s", err)
				continue
			}

			recipientHandles := make([]*string, 0, 10)
			for _, m := range messages {
				payloads[*m.Body] = struct{}{}
				recipientHandles = append(recipientHandles, m.ReceiptHandle)
			}

			if err := q.DeleteMessageBatch(recipientHandles...); err != nil {
				log.Printf("unable to delete message but continuing...: %s", err)
			}
		}

		log.Printf("%d messages received", config.NumberOfMessages)
		errCh <- nil
	}()

	select {
	case <-errCh:
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Unable to receive messages: %s", err))
			return 1
		}
		log.Print("bench: done.")
	case <-c.ShutdownCh:
		log.Fatal("signal received.")
	}

	return 0
}
Ejemplo n.º 5
0
func (c *BenchCommand) Send(args []string) int {
	var config BenchSendConfig

	if err := envconfig.Process("delayd2", &config); err != nil {
		c.Ui.Error(fmt.Sprintf("Unable to get configuration from envvars: %s", err))
		return 1
	}

	cmdFlags := flag.NewFlagSet("batch send", flag.ContinueOnError)
	cmdFlags.IntVar(&config.NumberOfMessages, "n", 1000, "number of messages")
	cmdFlags.IntVar(&config.Concurrency, "c", 5, "concurrency")
	cmdFlags.IntVar(&config.Duration, "d", 10, "duration in second")

	if err := cmdFlags.Parse(args); err != nil {
		c.Ui.Error(c.Help())
		return 1
	}

	sqsSvc := sqs.New(session.New())
	q, err := queue.New(sqsSvc, config.QueueName)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Unable to initialize SQS connection: %s", err))
		return 1
	}

	log.Print("bench: --- Configuration ---")
	log.Printf("bench: Number of messages to send: %d", config.NumberOfMessages)
	log.Printf("bench: Concurrency of sending: %d", config.Concurrency)
	log.Printf("bench: Duration of relaying: %d", config.Duration)
	log.Printf("bench: DELAYD2_QUEUE_NAME=%s", config.QueueName)
	log.Printf("bench: DELAYD2_TARGET_QUEUE_NAME=%s", config.TargetQueueName)

	s := delayd2.NewSender(q)

	log.Print("bench: starting")
	payloads := make([]string, config.NumberOfMessages)
	for i := 0; i < config.NumberOfMessages; i++ {
		payloads[i] = fmt.Sprintf("%d %d", i, time.Now().Unix())
	}

	tasks := DistributeN(config.Concurrency, payloads)

	var wg sync.WaitGroup
	wg.Add(len(tasks))

	begin := time.Now()
	for _, task := range tasks {
		go func(t []string) {
			defer wg.Done()

			log.Print("launching worker goroutine")
			if err := s.SendMessageBatch(config.Duration, config.TargetQueueName, t); err != nil {
				log.Print(err)
				return
			}
		}(task)
	}

	doneCh := make(chan struct{})
	go func() {
		wg.Wait()
		end := time.Now()
		log.Printf("%d messages sent in %s", len(payloads), end.Sub(begin))
		close(doneCh)
	}()

	select {
	case <-doneCh:
		log.Print("bench: done.")
	case <-c.ShutdownCh:
		c.Ui.Error("signal received")
		return 1
	}

	return 0
}