func (m *LoadBalancerMode) onMessage(client ProtocolClient, msg eventsMessage) {
if msg.event != nil {
if err := client.PublishEvent(msg.event); err != nil {
msg.attemptsLeft--
m.onFail(msg, err)
return
}
} else {
events := msg.events
total := len(events)
var backoffCount uint
for len(events) > 0 {
var err error
events, err = client.PublishEvents(events)
if err != nil {
msg.attemptsLeft--
// reset attempt count if subset of messages has been processed
if len(events) < total {
msg.attemptsLeft = m.maxAttempts
}
if err != ErrTempBulkFailure {
// retry non-published subset of events in batch
msg.events = events
m.onFail(msg, err)
return
}
if m.maxAttempts > 0 && msg.attemptsLeft <= 0 {
// no more attempts left => drop
outputs.SignalFailed(msg.signaler, err)
return
}
// wait before retry
backoff := time.Duration(int64(m.waitRetry) * (1 << backoffCount))
if backoff > m.maxWaitRetry {
backoff = m.maxWaitRetry
} else {
backoffCount++
}
select {
case <-m.done: // shutdown
outputs.SignalFailed(msg.signaler, err)
return
case <-time.After(backoff):
}
// reset total count for temporary failure loop
total = len(events)
}
}
}
outputs.SignalCompleted(msg.signaler)
}
func (s *singleConnectionMode) PublishEvents(
trans outputs.Signaler,
events []common.MapStr,
) error {
published := 0
fails := 0
for !s.closed && (s.maxAttempts == 0 || fails < s.maxAttempts) {
if err := s.Connect(); err != nil {
fails++
time.Sleep(s.waitRetry)
continue
}
for published < len(events) {
n, err := s.conn.PublishEvents(events[published:])
if err != nil {
break
}
fails = 0
published += n
}
if published == len(events) {
outputs.SignalCompleted(trans)
return nil
}
time.Sleep(s.waitRetry)
fails++
}
outputs.SignalFailed(trans)
return nil
}
func (mh *testMessageHandler) acknowledgeMessage(m message) {
if mh.response == CompletedResponse {
outputs.SignalCompleted(m.context.signal)
} else {
outputs.SignalFailed(m.context.signal, nil)
}
}
func (c *console) PublishEvent(
s outputs.Signaler,
ts time.Time,
event common.MapStr,
) error {
var jsonEvent []byte
var err error
if c.pretty {
jsonEvent, err = json.MarshalIndent(event, "", " ")
} else {
jsonEvent, err = json.Marshal(event)
}
if err != nil {
logp.Err("Fail to convert the event to JSON: %s", err)
outputs.SignalCompleted(s)
return err
}
if err = writeBuffer(jsonEvent); err != nil {
goto fail
}
if err = writeBuffer([]byte{'\n'}); err != nil {
goto fail
}
outputs.SignalCompleted(s)
return nil
fail:
logp.Err("Fail to write event: %s", err)
outputs.SignalFailed(s)
return err
}
// PublishEvent forwards a single event. On failure PublishEvent tries to reconnect.
func (f *FailOverConnectionMode) PublishEvent(
signaler outputs.Signaler,
event common.MapStr,
) error {
fails := 0
var err error
for !f.closed && (f.maxAttempts == 0 || fails < f.maxAttempts) {
if err := f.connect(f.active); err != nil {
logp.Info("Connecting error publishing events (retrying): %s", err)
fails++
time.Sleep(f.waitRetry)
continue
}
if err := f.conns[f.active].PublishEvent(event); err != nil {
logp.Info("Error publishing events (retrying): %s", err)
fails++
continue
}
outputs.SignalCompleted(signaler)
return nil
}
outputs.SignalFailed(signaler, err)
return nil
}
// PublishEvent forwards a single event. On failure PublishEvent tries to reconnect.
func (s *SingleConnectionMode) PublishEvent(
signaler outputs.Signaler,
event common.MapStr,
) error {
fails := 0
var err error
for !s.closed && (s.maxAttempts == 0 || fails < s.maxAttempts) {
if err = s.connect(); err != nil {
logp.Info("Connecting error publishing event (retrying): %s", err)
fails++
time.Sleep(s.waitRetry)
continue
}
if err := s.conn.PublishEvent(event); err != nil {
logp.Info("Error publishing event (retrying): %s", err)
fails++
continue
}
outputs.SignalCompleted(signaler)
return nil
}
outputs.SignalFailed(signaler, err)
return nil
}
func (m *LoadBalancerMode) onFail(msg eventsMessage, err error) {
logp.Info("Error publishing events (retrying): %s", err)
if ok := m.forwardEvent(m.retries, msg); !ok {
outputs.SignalFailed(msg.signaler, err)
}
}
func (out *redisOutput) BulkPublish(
signal outputs.Signaler,
ts time.Time,
events []common.MapStr,
) error {
if !out.connected {
logp.Debug("output_redis", "Droping pkt ...")
return errors.New("Not connected")
}
command := "RPUSH"
if out.DataType == RedisChannelType {
command = "PUBLISH"
}
if len(events) == 1 { // single event
event := events[0]
jsonEvent, err := json.Marshal(event)
if err != nil {
logp.Err("Fail to convert the event to JSON: %s", err)
outputs.SignalCompleted(signal)
return err
}
_, err = out.Conn.Do(command, out.Index, string(jsonEvent))
outputs.Signal(signal, err)
out.onFail(err)
return err
}
for _, event := range events {
jsonEvent, err := json.Marshal(event)
if err != nil {
logp.Err("Fail to convert the event to JSON: %s", err)
continue
}
err = out.Conn.Send(command, out.Index, string(jsonEvent))
if err != nil {
outputs.SignalFailed(signal, err)
out.onFail(err)
return err
}
}
if err := out.Conn.Flush(); err != nil {
outputs.Signal(signal, err)
out.onFail(err)
return err
}
_, err := out.Conn.Receive()
outputs.Signal(signal, err)
out.onFail(err)
return err
}
// publish is used to publish events using the configured protocol client.
// It provides general error handling and back off support used on failed
// send attempts. To be used by PublishEvent and PublishEvents.
// The send callback will try to progress sending traffic and returns kind of
// progress made in ok or resetFail. If ok is set to true, send finished
// processing events. If ok is false but resetFail is set, send was partially
// successful. If send was partially successful, the fail counter is reset thus up
// to maxAttempts send attempts without any progress might be executed.
func (s *SingleConnectionMode) publish(
signaler outputs.Signaler,
send func() (ok bool, resetFail bool),
) error {
fails := 0
var backoffCount uint
var err error
for !s.closed && (s.maxAttempts == 0 || fails < s.maxAttempts) {
ok := false
resetFail := false
if err := s.connect(); err != nil {
logp.Info("Connecting error publishing events (retrying): %s", err)
goto sendFail
}
ok, resetFail = send()
if !ok {
goto sendFail
}
outputs.SignalCompleted(signaler)
return nil
sendFail:
fails++
if resetFail {
fails = 0
}
if s.maxAttempts > 0 && fails == s.maxAttempts {
// max number of attempts reached
break
}
logp.Info("send fail")
backoff := time.Duration(int64(s.waitRetry) * (1 << backoffCount))
if backoff > s.maxWaitRetry {
backoff = s.maxWaitRetry
} else {
backoffCount++
}
logp.Info("backoff retry: %v", backoff)
time.Sleep(backoff)
}
outputs.SignalFailed(signaler, err)
return nil
}
func (m *loadBalancerMode) PublishEvents(
signaler outputs.Signaler,
events []common.MapStr,
) error {
msg := eventsMessage{
attemptsLeft: m.maxAttempts,
signaler: signaler,
events: events,
}
if ok := m.forwardEvent(m.work, msg); !ok {
outputs.SignalFailed(msg.signaler)
}
return nil
}
// PublishEvents tries to publish the events with retries if connection becomes
// unavailable. On failure PublishEvents tries to reconnect.
func (s *SingleConnectionMode) PublishEvents(
signaler outputs.Signaler,
events []common.MapStr,
) error {
fails := 0
var backoffCount uint
var err error
for !s.closed && (s.maxAttempts == 0 || fails < s.maxAttempts) {
if err := s.connect(); err != nil {
logp.Info("Connecting error publishing events (retrying): %s", err)
goto sendFail
}
for len(events) > 0 {
var err error
events, err = s.conn.PublishEvents(events)
if err != nil {
logp.Info("Error publishing events (retrying): %s", err)
break
}
fails = 0
}
if len(events) == 0 {
outputs.SignalCompleted(signaler)
return nil
}
sendFail:
logp.Info("send fail")
backoff := time.Duration(int64(s.waitRetry) * (1 << backoffCount))
if backoff > s.maxWaitRetry {
backoff = s.maxWaitRetry
} else {
backoffCount++
}
logp.Info("backoff retry: %v", backoff)
time.Sleep(backoff)
fails++
}
outputs.SignalFailed(signaler, err)
return nil
}
// PublishEvents tries to publish the events with retries if connection becomes
// unavailable. On failure PublishEvents tries to connect to another configured
// connection by random.
func (f *FailOverConnectionMode) PublishEvents(
trans outputs.Signaler,
events []common.MapStr,
) error {
published := 0
fails := 0
var err error
for !f.closed && (f.maxAttempts == 0 || fails < f.maxAttempts) {
if err = f.connect(f.active); err != nil {
logp.Info("Connecting error publishing events (retrying): %s", err)
fails++
time.Sleep(f.waitRetry)
continue
}
// loop until all events have been send in case client supports partial sends
for published < len(events) {
conn := f.conns[f.active]
n, err := conn.PublishEvents(events[published:])
if err != nil {
logp.Info("Error publishing events (retrying): %s", err)
break
}
published += n
}
if published == len(events) {
outputs.SignalCompleted(trans)
return nil
}
// TODO(sissel): Track how frequently we timeout and reconnect. If we're
// timing out too frequently, there's really no point in timing out since
// basically everything is slow or down. We'll want to ratchet up the
// timeout value slowly until things improve, then ratchet it down once
// things seem healthy.
time.Sleep(f.waitRetry)
fails++
}
outputs.SignalFailed(trans, err)
return nil
}
// publish is used to publish events using the configured protocol client.
// It provides general error handling and back off support used on failed
// send attempts. To be used by PublishEvent and PublishEvents.
// The send callback will try to progress sending traffic and returns kind of
// progress made in ok or resetFail. If ok is set to true, send finished
// processing events. If ok is false but resetFail is set, send was partially
// successful. If send was partially successful, the fail counter is reset thus up
// to maxAttempts send attempts without any progress might be executed.
func (f *FailOverConnectionMode) publish(
signaler outputs.Signaler,
send func() (ok bool, resetFail bool),
) error {
fails := 0
var err error
// TODO: we want back off support here? Fail over normally will try another
// connection.
for !f.closed && (f.maxAttempts == 0 || fails < f.maxAttempts) {
ok := false
resetFail := false
if err = f.connect(f.active); err != nil {
logp.Info("Connecting error publishing events (retrying): %s", err)
goto sendFail
}
ok, resetFail = send()
if !ok {
goto sendFail
}
outputs.SignalCompleted(signaler)
return nil
sendFail:
fails++
if resetFail {
fails = 0
}
if f.maxAttempts > 0 && fails == f.maxAttempts {
// max number of attempts reached
break
}
time.Sleep(f.waitRetry)
}
outputs.SignalFailed(signaler, err)
return nil
}
// PublishEvents tries to publish the events with retries if connection becomes
// unavailable. On failure PublishEvents tries to reconnect.
func (s *SingleConnectionMode) PublishEvents(
signaler outputs.Signaler,
events []common.MapStr,
) error {
published := 0
fails := 0
var err error
for !s.closed && (s.maxAttempts == 0 || fails < s.maxAttempts) {
if err := s.connect(); err != nil {
logp.Info("Connecting error publishing events (retrying): %s", err)
fails++
time.Sleep(s.waitRetry)
continue
}
for published < len(events) {
n, err := s.conn.PublishEvents(events[published:])
if err != nil {
logp.Info("Error publishing events (retrying): %s", err)
break
}
fails = 0
published += n
}
if published == len(events) {
outputs.SignalCompleted(signaler)
return nil
}
time.Sleep(s.waitRetry)
fails++
}
outputs.SignalFailed(signaler, err)
return nil
}
func stopQueue(qu chan message) {
close(qu)
for msg := range qu { // clear queue and send fail signal
outputs.SignalFailed(msg.context.signal, nil)
}
}
func (m *loadBalancerMode) onFail(msg eventsMessage) {
if ok := m.forwardEvent(m.retries, msg); !ok {
outputs.SignalFailed(msg.signaler)
}
}
func (m *LoadBalancerMode) publishEventsMessage(msg eventsMessage) error {
if ok := m.forwardEvent(m.work, msg); !ok {
outputs.SignalFailed(msg.signaler, nil)
}
return nil
}