Exemplo n.º 1
0
Arquivo: mq.go Projeto: romand/ironcli
// Will create a new queue, all fields are optional.
// Queue type cannot be changed.
func ConfigCreateQueue(queueInfo QueueInfo, settings *config.Settings) (QueueInfo, error) {
	if queueInfo.Name == "" {
		return QueueInfo{}, errors.New("Name of queue is empty")
	}

	url := api.Action(config.ManualConfig("iron_mq", settings), "queues", queueInfo.Name)

	in := struct {
		Queue QueueInfo `json:"queue"`
	}{
		Queue: queueInfo,
	}

	var out struct {
		Queue QueueInfo `json:"queue"`
	}

	err := url.Req("PUT", in, &out)
	return out.Queue, err
}
Exemplo n.º 2
0
Arquivo: mq.go Projeto: romand/ironcli
// ConfigNew uses the specified settings over configuration specified in an iron.json file or
// environment variables to return a Queue object capable of acquiring information about or
// modifying the queue specified by queueName.
func ConfigNew(queueName string, settings *config.Settings) Queue {
	return Queue{Settings: config.ManualConfig("iron_mq", settings), Name: queueName}
}
Exemplo n.º 3
0
func main() {
	start := time.Now()

	// Parse config json and validate
	worker.ParseFlags()
	c := DefaultConfig()
	worker.ConfigFromJSON(c)
	if err := c.Valid(); err != nil {
		log.Fatal(err)
	}

	// Copy config, obfuscate token, and print for record
	copy := *c
	copy.Env.Token = obfuscate(copy.Env.Token)
	j, err := json.MarshalIndent(copy, "", "    ")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(j))

	// Retrieve queue info
	s := config.ManualConfig("iron_mq", &c.Env)
	t := time.Now()
	q := mq.ConfigNew(c.QueueName, &s)
	info, err := q.Info()
	if err != nil {
		log.Fatal("Could not access queue info", err)
	}
	fmt.Printf("Queue has %d messages. (request took %v)\n", info.Size, time.Since(t))

	// Loop for multiple iterations of dequeuing & processing messages
	emptyResultCount := 0
	totalProcessedCount := 0
batchLoop:
	for x := 0; c.MaxIterations == nil || x < *c.MaxIterations; x++ {
		// Determine if we have time for another set of processing
		if time.Since(start) > c.MaxDuration {
			break
		}

		// If configured, sleep between iterations
		if x != 0 {
			fmt.Println("Sleeping", c.IterationSleep)
			time.Sleep(c.IterationSleep)
		}

		// Create a timeout that can accommodate the duration of the entire batch
		timeout := float64(c.BatchSize) * c.MsgDuration.Seconds() * 1.1 // give it an extra 10% time for safety

		// Reserve messages with given batch size, timeout, and longpoll time
		t = time.Now()
		msgs, err := q.LongPoll(c.BatchSize, int(timeout), c.DequeueWait, false)
		if err != nil {
			// Ideally, continue the loop and try again later. After a certain amount of time, do panic/Fatal
			log.Fatal(err)
		}
		fmt.Printf("Iteration %d: Requested %d, got %d (request took %v with a max wait of %ds)\n", x, c.BatchSize, len(msgs), time.Since(t), c.DequeueWait)

		// Handle case of zero messages
		if len(msgs) == 0 && c.MaxEmptyResults != nil {
			emptyResultCount++
			if emptyResultCount >= *c.MaxEmptyResults {
				fmt.Println("Queue is empty - breaking work loop")
				break
			}
		} else {
			// Reset count if queue isn't empty
			emptyResultCount = 0
		}

		// Process each message
		for i, msg := range msgs {
			// Determine if we have time to process another message
			if time.Since(start) > c.MaxDuration+c.MsgDuration {
				fmt.Println("Not enough time to process message - breaking work loop")
				break batchLoop
			}

			// Simulate Message Processing
			time.Sleep(c.MsgDuration)
			fmt.Printf(" %d: %q\n", i, msg.Body)

			// Example case for error processing
			var processingError error
			if processingError != nil {
				// Move error to error queue
				// errorQueue.Push(msg)
			}

			totalProcessedCount++

			// Delete processed message from queue
			err = msg.Delete()
			if err != nil {
				fmt.Println("Could not delete msg:", msg.Body, err)
			}
		}
	}

	fmt.Printf("Worker ending after %s and processing %d messages", time.Since(start), totalProcessedCount)
}