Example #1
0
func (b *bulkWorker) onEvents(signal outputs.Signaler, events []common.MapStr) {
	for len(events) > 0 {
		// split up bulk to match required bulk sizes.
		// If input events have been split up bufferFull will be set and
		// bulk request will be published.
		bufferFull := false
		spaceLeft := cap(b.events) - len(b.events)
		consume := len(events)
		if spaceLeft < consume {
			bufferFull = true
			consume = spaceLeft
			if signal != nil {
				// creating cascading signaler chain for
				// subset of events being send
				signal = outputs.NewSplitSignaler(signal, 2)
			}
		}

		// buffer events
		b.events = append(b.events, events[:consume]...)
		events = events[consume:]
		if signal != nil {
			b.pending = append(b.pending, signal)
		}

		if bufferFull {
			b.publish()
		}
	}
}
Example #2
0
func (p *syncPublisher) onMessage(m message) {
	signal := outputs.NewSplitSignaler(m.context.signal, len(p.pub.Output))
	m.context.signal = signal
	for _, o := range p.pub.Output {
		o.send(m)
	}
}
Example #3
0
func (o *outputWorker) onBulk(ctx *context, events []common.MapStr) {
	if len(events) == 0 {
		debug("output worker: no events to publish")
		outputs.SignalCompleted(ctx.signal)
		return
	}

	var sync *outputs.SyncSignal
	if ctx.sync {
		sync = outputs.NewSyncSignal()
	}

	if o.maxBulkSize < 0 || len(events) <= o.maxBulkSize {
		o.sendBulk(sync, ctx, events)
		return
	}

	// start splitting bulk request
	splits := (len(events) + (o.maxBulkSize - 1)) / o.maxBulkSize
	ctx.signal = outputs.NewSplitSignaler(ctx.signal, splits)
	for len(events) > 0 {
		sz := o.maxBulkSize
		if sz > len(events) {
			sz = len(events)
		}
		o.sendBulk(sync, ctx, events[:sz])
		events = events[sz:]
	}
}
Example #4
0
func (p *asyncPublisher) onMessage(m message) {
	debug("async forward to outputers (%v)", len(p.outputs))

	// m.signal is not set yet. But a async client type supporting signals might
	// be implemented in the furute.
	// If m.signal is nil, NewSplitSignaler will return nil -> signaler will
	// only set if client did send one
	if m.context.signal != nil && len(p.outputs) > 1 {
		m.context.signal = outputs.NewSplitSignaler(m.context.signal, len(p.outputs))
	}
	for _, o := range p.outputs {
		o.send(m)
	}
}