Example #1
0
func (s *Sender) SendMessageBatch(duration int, relayTo string, payloads []string) error {
	attrs := map[string]interface{}{
		sqsMessageDurationKey: duration,
		sqsMessageRelayToKey:  relayTo,
	}

	remaining := payloads
	var cur []string
	for {
		if len(remaining) == 0 {
			break
		}

		if len(remaining) > 10 {
			cur = remaining[:10]
		} else {
			cur = remaining
		}

		batch := make([]queue.BatchMessage, 0, len(cur))

		for _, body := range cur {
			batch = append(batch, queue.BatchMessage{
				Body:    body,
				Options: []option.SendMessageInput{option.MessageAttributes(attrs)},
			})
		}

		failedIndex := make(map[int]struct{})
		if err := s.queue.SendMessageBatch(batch...); err != nil {
			berrs, batchErr := queue.IsBatchError(err)
			if !batchErr {
				// retry until it succeeds
				log.Printf("unable to send messages but continuing: %s", err)
				continue
			}

			for _, berr := range berrs {
				if berr.SenderFault {
					// return immediately if error is on our fault
					return err
				}
				failedIndex[berr.Index] = struct{}{}
			}
		}
		remaining = remaining[len(cur):]

		// restore failed messages
		for i, m := range batch {
			if _, failed := failedIndex[i]; failed {
				remaining = append(remaining, m.Body)
			}
		}
	}

	return nil
}
Example #2
0
func (w *Worker) releaseBatch(rj releaseJob) int64 {
	payloads := make([]string, 0, len(rj.Messages))
	for _, m := range rj.Messages {
		if _, found := w.succededIDsCache.Get(m.QueueID); found {
			// skip since we already relayed
			log.Printf("worker: seeing queue_id in the cache so continueing...")
			continue
		}
		payloads = append(payloads, m.Payload)
	}

	var n int64
	failedIndex := make(map[int]struct{})
	if len(payloads) > 0 {
		if err := w.relay.Relay(rj.RelayTo, payloads); err != nil {
			// only print log here in batch operation
			// TODO: a dead letter queue support
			berrs, batchOk := queue.IsBatchError(err)
			if !batchOk {
				log.Printf("worker: unable to send message due to non batch error. skipping: %s", err)
				return 0
			}

			for _, berr := range berrs {
				if berr.SenderFault {
					log.Printf("worker: unable to send message due to sender's fault: skipping: %s", berr.Message)
				}
				failedIndex[berr.Index] = struct{}{}
			}
		}
	}

	succededIDs := make([]string, 0, len(rj.Messages))
	for i, m := range rj.Messages {
		if _, failed := failedIndex[i]; failed {
			continue
		}
		succededIDs = append(succededIDs, m.QueueID)

		// remember succededIDs for a while to prevent us from relaying message in transient error
		w.succededIDsCache.Set(m.QueueID, struct{}{}, cache.DefaultExpiration)
	}

	if err := w.driver.RemoveMessages(succededIDs); err != nil {
		log.Printf("worker: unable to remove messages from queue: %s", err)
	} else {
		// reset succededIDs here since we succeeded to remove messages from the database
		// When we succeeded to remove messages, the messages will not appear again so it's safe.
		for _, id := range succededIDs {
			w.succededIDsCache.Delete(id)
		}
		n += int64(len(succededIDs))
	}

	return n
}