func (s *Sender) SendMessageBatch(duration int, relayTo string, payloads []string) error { attrs := map[string]interface{}{ sqsMessageDurationKey: duration, sqsMessageRelayToKey: relayTo, } remaining := payloads var cur []string for { if len(remaining) == 0 { break } if len(remaining) > 10 { cur = remaining[:10] } else { cur = remaining } batch := make([]queue.BatchMessage, 0, len(cur)) for _, body := range cur { batch = append(batch, queue.BatchMessage{ Body: body, Options: []option.SendMessageInput{option.MessageAttributes(attrs)}, }) } failedIndex := make(map[int]struct{}) if err := s.queue.SendMessageBatch(batch...); err != nil { berrs, batchErr := queue.IsBatchError(err) if !batchErr { // retry until it succeeds log.Printf("unable to send messages but continuing: %s", err) continue } for _, berr := range berrs { if berr.SenderFault { // return immediately if error is on our fault return err } failedIndex[berr.Index] = struct{}{} } } remaining = remaining[len(cur):] // restore failed messages for i, m := range batch { if _, failed := failedIndex[i]; failed { remaining = append(remaining, m.Body) } } } return nil }
func (w *Worker) releaseBatch(rj releaseJob) int64 { payloads := make([]string, 0, len(rj.Messages)) for _, m := range rj.Messages { if _, found := w.succededIDsCache.Get(m.QueueID); found { // skip since we already relayed log.Printf("worker: seeing queue_id in the cache so continueing...") continue } payloads = append(payloads, m.Payload) } var n int64 failedIndex := make(map[int]struct{}) if len(payloads) > 0 { if err := w.relay.Relay(rj.RelayTo, payloads); err != nil { // only print log here in batch operation // TODO: a dead letter queue support berrs, batchOk := queue.IsBatchError(err) if !batchOk { log.Printf("worker: unable to send message due to non batch error. skipping: %s", err) return 0 } for _, berr := range berrs { if berr.SenderFault { log.Printf("worker: unable to send message due to sender's fault: skipping: %s", berr.Message) } failedIndex[berr.Index] = struct{}{} } } } succededIDs := make([]string, 0, len(rj.Messages)) for i, m := range rj.Messages { if _, failed := failedIndex[i]; failed { continue } succededIDs = append(succededIDs, m.QueueID) // remember succededIDs for a while to prevent us from relaying message in transient error w.succededIDsCache.Set(m.QueueID, struct{}{}, cache.DefaultExpiration) } if err := w.driver.RemoveMessages(succededIDs); err != nil { log.Printf("worker: unable to remove messages from queue: %s", err) } else { // reset succededIDs here since we succeeded to remove messages from the database // When we succeeded to remove messages, the messages will not appear again so it's safe. for _, id := range succededIDs { w.succededIDsCache.Delete(id) } n += int64(len(succededIDs)) } return n }