func (p *syncPublisher) send(m message) bool { if p.pub.disabled { debug("publisher disabled") outputs.SignalCompleted(m.context.signal) return true } signal := m.context.signal sync := outputs.NewSyncSignal() if len(p.pub.Output) > 1 { m.context.signal = outputs.NewSplitSignaler(sync, len(p.pub.Output)) } else { m.context.signal = sync } for _, o := range p.pub.Output { o.send(m) } ok := sync.Wait() if ok { outputs.SignalCompleted(signal) } else if signal != nil { signal.Failed() } return ok }
func (c *console) PublishEvent( s outputs.Signaler, opts outputs.Options, event common.MapStr, ) error { var jsonEvent []byte var err error if c.config.Pretty { jsonEvent, err = json.MarshalIndent(event, "", " ") } else { jsonEvent, err = json.Marshal(event) } if err != nil { logp.Err("Fail to convert the event to JSON: %s", err) outputs.SignalCompleted(s) return err } if err = writeBuffer(jsonEvent); err != nil { goto fail } if err = writeBuffer([]byte{'\n'}); err != nil { goto fail } outputs.SignalCompleted(s) return nil fail: if opts.Guaranteed { logp.Critical("Unable to publish events to console: %v", err) } outputs.SignalFailed(s, err) return err }
func (c *console) PublishEvent( s outputs.Signaler, ts time.Time, event common.MapStr, ) error { var jsonEvent []byte var err error if c.pretty { jsonEvent, err = json.MarshalIndent(event, "", " ") } else { jsonEvent, err = json.Marshal(event) } if err != nil { logp.Err("Fail to convert the event to JSON: %s", err) outputs.SignalCompleted(s) return err } if err = writeBuffer(jsonEvent); err != nil { goto fail } if err = writeBuffer([]byte{'\n'}); err != nil { goto fail } outputs.SignalCompleted(s) return nil fail: outputs.SignalFailed(s, err) return err }
func (out *fileOutput) PublishEvent( trans outputs.Signaler, opts outputs.Options, event common.MapStr, ) error { jsonEvent, err := json.Marshal(event) if err != nil { // mark as success so event is not sent again. outputs.SignalCompleted(trans) logp.Err("Fail to convert the event to JSON: %s", err) return err } err = out.rotator.WriteLine(jsonEvent) if err != nil { if opts.Guaranteed { logp.Critical("Unable to write events to file: %s", err) } else { logp.Err("Error when writing line to file: %s", err) } } outputs.Signal(trans, err) return err }
func (mh *testMessageHandler) acknowledgeMessage(m message) { if mh.response == CompletedResponse { outputs.SignalCompleted(m.context.signal) } else { outputs.SignalFailed(m.context.signal, nil) } }
func (o *outputWorker) onBulk(ctx *context, events []common.MapStr) { if len(events) == 0 { debug("output worker: no events to publish") outputs.SignalCompleted(ctx.signal) return } var sync *outputs.SyncSignal if ctx.sync { sync = outputs.NewSyncSignal() } if o.maxBulkSize < 0 || len(events) <= o.maxBulkSize { o.sendBulk(sync, ctx, events) return } // start splitting bulk request splits := (len(events) + (o.maxBulkSize - 1)) / o.maxBulkSize ctx.signal = outputs.NewSplitSignaler(ctx.signal, splits) for len(events) > 0 { sz := o.maxBulkSize if sz > len(events) { sz = len(events) } o.sendBulk(sync, ctx, events[:sz]) events = events[sz:] } }
// publish is used to publish events using the configured protocol client. // It provides general error handling and back off support used on failed // send attempts. To be used by PublishEvent and PublishEvents. // The send callback will try to progress sending traffic and returns kind of // progress made in ok or resetFail. If ok is set to true, send finished // processing events. If ok is false but resetFail is set, send was partially // successful. If send was partially successful, the fail counter is reset thus up // to maxAttempts send attempts without any progress might be executed. func (s *SingleConnectionMode) publish( signaler outputs.Signaler, opts outputs.Options, send func() (ok bool, resetFail bool), ) error { fails := 0 var backoffCount uint var err error guaranteed := opts.Guaranteed || s.maxAttempts == 0 for !s.closed && (guaranteed || fails < s.maxAttempts) { ok := false resetFail := false if err := s.connect(); err != nil { logp.Info("Connecting error publishing events (retrying): %s", err) goto sendFail } ok, resetFail = send() if !ok { goto sendFail } debug("send completed") outputs.SignalCompleted(signaler) return nil sendFail: fails++ if resetFail { debug("reset fails") fails = 0 } if !guaranteed && (s.maxAttempts > 0 && fails == s.maxAttempts) { // max number of attempts reached debug("max number of attempts reached") break } logp.Info("send fail") backoff := time.Duration(int64(s.waitRetry) * (1 << backoffCount)) if backoff > s.maxWaitRetry { backoff = s.maxWaitRetry } else { backoffCount++ } logp.Info("backoff retry: %v", backoff) time.Sleep(backoff) } debug("messages dropped") messagesDropped.Add(1) outputs.SignalFailed(signaler, err) return nil }
func (m *LoadBalancerMode) onMessage(client ProtocolClient, msg eventsMessage) { if msg.event != nil { if err := client.PublishEvent(msg.event); err != nil { msg.attemptsLeft-- m.onFail(msg, err) return } } else { events := msg.events total := len(events) var backoffCount uint for len(events) > 0 { var err error events, err = client.PublishEvents(events) if err != nil { msg.attemptsLeft-- // reset attempt count if subset of messages has been processed if len(events) < total { msg.attemptsLeft = m.maxAttempts } if err != ErrTempBulkFailure { // retry non-published subset of events in batch msg.events = events m.onFail(msg, err) return } if m.maxAttempts > 0 && msg.attemptsLeft <= 0 { // no more attempts left => drop dropping(msg) return } // wait before retry backoff := time.Duration(int64(m.waitRetry) * (1 << backoffCount)) if backoff > m.maxWaitRetry { backoff = m.maxWaitRetry } else { backoffCount++ } select { case <-m.done: // shutdown outputs.SignalFailed(msg.signaler, err) return case <-time.After(backoff): } // reset total count for temporary failure loop total = len(events) } } } outputs.SignalCompleted(msg.signaler) }
func handlePublishEventsResult( m *AsyncLoadBalancerMode, msg eventsMessage, ) func([]common.MapStr, error) { total := len(msg.events) return func(events []common.MapStr, err error) { debug("handlePublishEventsResult") if err != nil { debug("handle publish error: %v", err) if msg.attemptsLeft > 0 { msg.attemptsLeft-- } // reset attempt count if subset of messages has been processed if len(events) < total && msg.attemptsLeft >= 0 { msg.attemptsLeft = m.maxAttempts } if err != ErrTempBulkFailure { // retry non-published subset of events in batch msg.events = events m.onFail(false, msg, err) return } if m.maxAttempts > 0 && msg.attemptsLeft == 0 { // no more attempts left => drop dropping(msg, &m.ws) return } // retry non-published subset of events in batch msg.events = events m.onFail(false, msg, err) return } // re-insert non-published events into pipeline if len(events) != 0 { debug("add non-published events back into pipeline: %v", len(events)) msg.events = events if ok := m.forwardEvent(m.retries, msg); !ok { dropping(msg, &m.ws) } return } // all events published -> signal success debug("async bulk publish success") outputs.SignalCompleted(msg.signaler) m.ws.DoneEvent() } }
func handlePublishEventResult(m *AsyncLoadBalancerMode, msg eventsMessage) func(error) { return func(err error) { if err != nil { if msg.attemptsLeft > 0 { msg.attemptsLeft-- } m.onFail(false, msg, err) } else { outputs.SignalCompleted(msg.signaler) } } }
func (out *redisOutput) BulkPublish( signal outputs.Signaler, ts time.Time, events []common.MapStr, ) error { if !out.connected { logp.Debug("output_redis", "Droping pkt ...") return errors.New("Not connected") } command := "RPUSH" if out.DataType == RedisChannelType { command = "PUBLISH" } if len(events) == 1 { // single event event := events[0] jsonEvent, err := json.Marshal(event) if err != nil { logp.Err("Fail to convert the event to JSON: %s", err) outputs.SignalCompleted(signal) return err } _, err = out.Conn.Do(command, out.Index, string(jsonEvent)) outputs.Signal(signal, err) out.onFail(err) return err } for _, event := range events { jsonEvent, err := json.Marshal(event) if err != nil { logp.Err("Fail to convert the event to JSON: %s", err) continue } err = out.Conn.Send(command, out.Index, string(jsonEvent)) if err != nil { outputs.SignalFailed(signal, err) out.onFail(err) return err } } if err := out.Conn.Flush(); err != nil { outputs.Signal(signal, err) out.onFail(err) return err } _, err := out.Conn.Receive() outputs.Signal(signal, err) out.onFail(err) return err }
// publish is used to publish events using the configured protocol client. // It provides general error handling and back off support used on failed // send attempts. To be used by PublishEvent and PublishEvents. // The send callback will try to progress sending traffic and returns kind of // progress made in ok or resetFail. If ok is set to true, send finished // processing events. If ok is false but resetFail is set, send was partially // successful. If send was partially successful, the fail counter is reset thus up // to maxAttempts send attempts without any progress might be executed. func (s *SingleConnectionMode) publish( signaler outputs.Signaler, opts outputs.Options, send func() (ok bool, resetFail bool), ) error { fails := 0 var err error guaranteed := opts.Guaranteed || s.maxAttempts == 0 for !s.closed && (guaranteed || fails < s.maxAttempts) { ok := false resetFail := false if err := s.connect(); err != nil { logp.Info("Connecting error publishing events (retrying): %s", err) goto sendFail } ok, resetFail = send() if !ok { goto sendFail } debug("send completed") s.backoff.Reset() outputs.SignalCompleted(signaler) return nil sendFail: logp.Info("send fail") s.backoff.Wait() fails++ if resetFail { debug("reset fails") fails = 0 } if !guaranteed && (s.maxAttempts > 0 && fails == s.maxAttempts) { // max number of attempts reached debug("max number of attempts reached") break } } debug("messages dropped") messagesDropped.Add(1) outputs.SignalFailed(signaler, err) return nil }
// publish is used to publish events using the configured protocol client. // It provides general error handling and back off support used on failed // send attempts. To be used by PublishEvent and PublishEvents. // The send callback will try to progress sending traffic and returns kind of // progress made in ok or resetFail. If ok is set to true, send finished // processing events. If ok is false but resetFail is set, send was partially // successful. If send was partially successful, the fail counter is reset thus up // to maxAttempts send attempts without any progress might be executed. func (f *FailOverConnectionMode) publish( signaler outputs.Signaler, opts outputs.Options, send func() (ok bool, resetFail bool), ) error { fails := 0 var err error // TODO: we want back off support here? Fail over normally will try another // connection. guaranteed := opts.Guaranteed || f.maxAttempts == 0 for !f.closed && (guaranteed || fails < f.maxAttempts) { ok := false resetFail := false if err = f.connect(f.active); err != nil { logp.Info("Connecting error publishing events (retrying): %s", err) goto sendFail } ok, resetFail = send() if !ok { goto sendFail } debug("send completed") outputs.SignalCompleted(signaler) return nil sendFail: fails++ if resetFail { debug("reset fails") fails = 0 } if !guaranteed && (f.maxAttempts > 0 && fails == f.maxAttempts) { // max number of attempts reached debug("max number of attempts reached") break } time.Sleep(f.waitRetry) } debug("messages dropped") messagesDropped.Add(1) outputs.SignalFailed(signaler, err) return nil }
func TestPublisherModes(t *testing.T) { tests := []struct { title string async bool order []int }{ {"sync", false, []int{1, 2, 3, 4, 5, 6}}, {"async ordered signal", true, []int{1, 2, 3, 4, 5, 6}}, {"async out of order signal", true, []int{5, 2, 3, 1, 4, 6}}, } for i, test := range tests { t.Logf("run publisher test (%v): %v", i, test.title) pubChan := make(chan []*input.FileEvent, len(test.order)+1) regChan := make(chan []*input.FileEvent, len(test.order)+1) client := publisher.ExtChanClient{make(chan publisher.PublishMessage)} pub := newPublisher(test.async, pubChan, regChan, client) pub.Start() var events [][]*input.FileEvent for i := range test.order { tmp := makeEvents(fmt.Sprintf("msg: %v", i), 1) pubChan <- tmp events = append(events, tmp) } var msgs []publisher.PublishMessage for _ = range test.order { m := <-client.Channel msgs = append(msgs, m) } for _, i := range test.order { outputs.SignalCompleted(msgs[i-1].Context.Signal) } var regEvents [][]*input.FileEvent for _ = range test.order { regEvents = append(regEvents, <-regChan) } pub.Stop() // validate order for i := range events { assert.Equal(t, events[i], regEvents[i]) } } }
func (p *syncPublisher) forward(m message) bool { sync := outputs.NewSyncSignal() signal := m.context.signal m.context.signal = sync p.send(m) if sync.Wait() { outputs.SignalCompleted(signal) return true } if signal != nil { signal.Failed() } return false }
// publish is used to publish events using the configured protocol client. // It provides general error handling and back off support used on failed // send attempts. To be used by PublishEvent and PublishEvents. // The send callback will try to progress sending traffic and returns kind of // progress made in ok or resetFail. If ok is set to true, send finished // processing events. If ok is false but resetFail is set, send was partially // successful. If send was partially successful, the fail counter is reset thus up // to maxAttempts send attempts without any progress might be executed. func (s *SingleConnectionMode) publish( signaler outputs.Signaler, send func() (ok bool, resetFail bool), ) error { fails := 0 var backoffCount uint var err error for !s.closed && (s.maxAttempts == 0 || fails < s.maxAttempts) { ok := false resetFail := false if err := s.connect(); err != nil { logp.Info("Connecting error publishing events (retrying): %s", err) goto sendFail } ok, resetFail = send() if !ok { goto sendFail } outputs.SignalCompleted(signaler) return nil sendFail: fails++ if resetFail { fails = 0 } if s.maxAttempts > 0 && fails == s.maxAttempts { // max number of attempts reached break } logp.Info("send fail") backoff := time.Duration(int64(s.waitRetry) * (1 << backoffCount)) if backoff > s.maxWaitRetry { backoff = s.maxWaitRetry } else { backoffCount++ } logp.Info("backoff retry: %v", backoff) time.Sleep(backoff) } outputs.SignalFailed(signaler, err) return nil }
func (p *asyncPublisher) send(m message) { if p.pub.disabled { debug("publisher disabled") outputs.SignalCompleted(m.context.signal) return } // m.signal is not set yet. But a async client type supporting signals might // be implemented in the future. // If m.signal is nil, NewSplitSignaler will return nil -> signaler will // only set if client did send one if m.context.signal != nil && len(p.outputs) > 1 { m.context.signal = outputs.NewSplitSignaler(m.context.signal, len(p.outputs)) } for _, o := range p.outputs { o.send(m) } }
func (o *outputWorker) onEvent(ctx *context, event common.MapStr) { debug("output worker: publish single event") ts := time.Time(event["@timestamp"].(common.Time)).UTC() if !ctx.sync { _ = o.out.PublishEvent(ctx.signal, ts, event) return } signal := outputs.NewSyncSignal() for { o.out.PublishEvent(signal, ts, event) if signal.Wait() { outputs.SignalCompleted(ctx.signal) break } } }
// publish is used to publish events using the configured protocol client. // It provides general error handling and back off support used on failed // send attempts. To be used by PublishEvent and PublishEvents. // The send callback will try to progress sending traffic and returns kind of // progress made in ok or resetFail. If ok is set to true, send finished // processing events. If ok is false but resetFail is set, send was partially // successful. If send was partially successful, the fail counter is reset thus up // to maxAttempts send attempts without any progress might be executed. func (f *FailOverConnectionMode) publish( signaler outputs.Signaler, send func() (ok bool, resetFail bool), ) error { fails := 0 var err error // TODO: we want back off support here? Fail over normally will try another // connection. for !f.closed && (f.maxAttempts == 0 || fails < f.maxAttempts) { ok := false resetFail := false if err = f.connect(f.active); err != nil { logp.Info("Connecting error publishing events (retrying): %s", err) goto sendFail } ok, resetFail = send() if !ok { goto sendFail } outputs.SignalCompleted(signaler) return nil sendFail: fails++ if resetFail { fails = 0 } if f.maxAttempts > 0 && fails == f.maxAttempts { // max number of attempts reached break } time.Sleep(f.waitRetry) } outputs.SignalFailed(signaler, err) return nil }
func (c *cmp) PublishEvent( s outputs.Signaler, opts outputs.Options, event common.MapStr, ) error { var err error if err = postToCmp(c, event); err != nil { goto fail } outputs.SignalCompleted(s) return nil fail: if opts.Guaranteed { logp.Critical("Unable to publish events to cmp: %v", err) } outputs.SignalFailed(s, err) return err }
func (o *outputWorker) sendBulk( sync *outputs.SyncSignal, ctx *context, events []common.MapStr, ) { debug("output worker: publish %v events", len(events)) ts := time.Time(events[0]["@timestamp"].(common.Time)).UTC() if sync == nil { err := o.out.BulkPublish(ctx.signal, ts, events) if err != nil { logp.Info("Error bulk publishing events: %s", err) } return } for done := false; !done; done = sync.Wait() { o.out.BulkPublish(sync, ts, events) } outputs.SignalCompleted(ctx.signal) }
func (out *redisOutput) BulkPublish( signal outputs.Signaler, opts outputs.Options, events []common.MapStr, ) error { if !opts.Guaranteed { err := out.doBulkPublish(events) outputs.Signal(signal, err) return err } for { err := out.doBulkPublish(events) if err == nil { outputs.SignalCompleted(signal) return nil } // TODO: add backoff time.Sleep(1) } }
func (out *fileOutput) PublishEvent( trans outputs.Signaler, ts time.Time, event common.MapStr, ) error { jsonEvent, err := json.Marshal(event) if err != nil { // mark as success so event is not sent again. outputs.SignalCompleted(trans) logp.Err("Fail to convert the event to JSON: %s", err) return err } err = out.rotator.WriteLine(jsonEvent) if err != nil { logp.Err("Error when writing line to file: %s", err) } outputs.Signal(trans, err) return err }
func (m *LoadBalancerMode) onMessage( backoff *backoff, client ProtocolClient, msg eventsMessage, ) (bool, error) { done := false if msg.event != nil { err := client.PublishEvent(msg.event) done = !backoff.WaitOnError(err) if err != nil { if msg.attemptsLeft > 0 { msg.attemptsLeft-- } m.onFail(msg, err) return done, err } } else { events := msg.events total := len(events) for len(events) > 0 { var err error events, err = client.PublishEvents(events) done = !backoff.WaitOnError(err) if done && err != nil { outputs.SignalFailed(msg.signaler, err) return done, err } if err != nil { if msg.attemptsLeft > 0 { msg.attemptsLeft-- } // reset attempt count if subset of messages has been processed if len(events) < total && msg.attemptsLeft >= 0 { debug("reset fails") msg.attemptsLeft = m.maxAttempts } if err != ErrTempBulkFailure { // retry non-published subset of events in batch msg.events = events m.onFail(msg, err) return done, err } if m.maxAttempts > 0 && msg.attemptsLeft == 0 { // no more attempts left => drop dropping(msg) return done, err } // reset total count for temporary failure loop total = len(events) } } } outputs.SignalCompleted(msg.signaler) return done, nil }
func (p *preprocessor) onMessage(m message) { publisher := p.pub single := false events := m.events if m.event != nil { single = true events = []common.MapStr{m.event} } var ignore []int // indices of events to be removed from events debug("Start Preprocessing") for i, event := range events { // validate some required field if err := filterEvent(event); err != nil { logp.Err("Publishing event failed: %v", err) ignore = append(ignore, i) continue } // update address and geo-ip information. Ignore event // if address is invalid or event is found to be a duplicate ok := updateEventAddresses(publisher, event) if !ok { ignore = append(ignore, i) continue } // add additional Beat meta data event["beat"] = common.MapStr{ "name": publisher.name, "hostname": publisher.hostname, } if len(publisher.tags) > 0 { event["tags"] = publisher.tags } if logp.IsDebug("publish") { PrintPublishEvent(event) } } // return if no event is left if len(ignore) == len(events) { debug("no event left, complete send") outputs.SignalCompleted(m.context.signal) return } // remove invalid events. // TODO: is order important? Removal can be turned into O(len(ignore)) by // copying last element into idx and doing // events=events[:len(events)-len(ignore)] afterwards // Alternatively filtering could be implemented like: // https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating for i := len(ignore) - 1; i >= 0; i-- { idx := ignore[i] debug("remove event[%v]", idx) events = append(events[:idx], events[idx+1:]...) } if publisher.disabled { debug("publisher disabled") outputs.SignalCompleted(m.context.signal) return } debug("Forward preprocessed events") if single { p.handler.onMessage(message{context: m.context, event: events[0]}) } else { p.handler.onMessage(message{context: m.context, events: events}) } }
// PublishEvent writes events to a channel then calls Completed on trans. // It always returns nil. func (t *testOutputer) PublishEvent(trans outputs.Signaler, ts time.Time, event common.MapStr) error { t.events <- event outputs.SignalCompleted(trans) return nil }