Пример #1
0
func (o *outputWorker) onBulk(ctx *context, events []common.MapStr) {
	if len(events) == 0 {
		debug("output worker: no events to publish")
		outputs.SignalCompleted(ctx.signal)
		return
	}

	var sync *outputs.SyncSignal
	if ctx.sync {
		sync = outputs.NewSyncSignal()
	}

	if o.maxBulkSize < 0 || len(events) <= o.maxBulkSize {
		o.sendBulk(sync, ctx, events)
		return
	}

	// start splitting bulk request
	splits := (len(events) + (o.maxBulkSize - 1)) / o.maxBulkSize
	ctx.signal = outputs.NewSplitSignaler(ctx.signal, splits)
	for len(events) > 0 {
		sz := o.maxBulkSize
		if sz > len(events) {
			sz = len(events)
		}
		o.sendBulk(sync, ctx, events[:sz])
		events = events[sz:]
	}
}
func testSendMultipleBatchesViaLogstash(
	t *testing.T,
	name string,
	numBatches int,
	batchSize int,
	tls bool,
) {
	if testing.Short() {
		t.Skip("Skipping in short mode. Requires Logstash and Elasticsearch")
	}

	ls := newTestLogstashOutput(t, name, tls)
	defer ls.Cleanup()

	batches := make([][]common.MapStr, 0, numBatches)
	for i := 0; i < numBatches; i++ {
		batch := make([]common.MapStr, 0, batchSize)
		for j := 0; j < batchSize; j++ {
			event := common.MapStr{
				"@timestamp": common.Time(time.Now()),
				"host":       "test-host",
				"type":       "log",
				"message":    fmt.Sprintf("batch hello world - %v", i*batchSize+j),
			}
			batch = append(batch, event)
		}
		batches = append(batches, batch)
	}

	for _, batch := range batches {
		sig := outputs.NewSyncSignal()
		ls.BulkPublish(sig, time.Now(), batch)
		ok := sig.Wait()
		assert.Equal(t, true, ok)
	}

	// wait for logstash event flush + elasticsearch
	ok := waitUntilTrue(5*time.Second, checkIndex(ls, numBatches*batchSize))
	assert.True(t, ok) // check number of events matches total number of events

	// search value in logstash elasticsearch index
	resp, err := ls.Read()
	if err != nil {
		return
	}
	if len(resp) != 10 {
		t.Errorf("wrong number of results: %d", len(resp))
	}
}
Пример #3
0
func (o *outputWorker) onEvent(ctx *context, event common.MapStr) {
	debug("output worker: publish single event")
	ts := time.Time(event["@timestamp"].(common.Time)).UTC()

	if !ctx.sync {
		_ = o.out.PublishEvent(ctx.signal, ts, event)
		return
	}

	signal := outputs.NewSyncSignal()
	for {
		o.out.PublishEvent(signal, ts, event)
		if signal.Wait() {
			outputs.SignalCompleted(ctx.signal)
			break
		}
	}
}
Пример #4
0
func TestLogstashInvalidTLS(t *testing.T) {
	certName := "ca_invalid_test"
	ip := net.IP{1, 2, 3, 4}

	timeout := 2 * time.Second
	genCertsForIPIfMIssing(t, ip, certName)
	server := newMockTLSServer(t, timeout, certName)

	retries := 0
	config := outputs.MothershipConfig{
		TLS: &outputs.TLSConfig{
			CAs: []string{certName + ".pem"},
		},
		Timeout:    1,
		MaxRetries: &retries,
		Hosts:      []string{server.Addr()},
	}

	var result struct {
		err           error
		handshakeFail bool
		signal        bool
	}

	var wg struct {
		ready  sync.WaitGroup
		finish sync.WaitGroup
	}

	wg.ready.Add(1)  // server signaling readiness to client worker
	wg.finish.Add(2) // server/client signaling test end

	// server loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Done()

		client := server.accept()
		if server.err != nil {
			t.Fatalf("server error: %v", server.err)
		}

		server.handshake(client)
		result.handshakeFail = server.err != nil
	}()

	// client loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Wait()

		output := newTestLumberjackOutput(t, "", &config)

		signal := outputs.NewSyncSignal()
		output.PublishEvent(signal, time.Now(), testEvent())
		result.signal = signal.Wait()
	}()

	// wait shutdown
	wg.finish.Wait()
	server.Close()

	// validate output
	assert.True(t, result.handshakeFail)
	assert.False(t, result.signal)
}
Пример #5
0
func testConnectionType(
	t *testing.T,
	server *mockLSServer,
	makeOutputer func() outputs.BulkOutputer,
) {
	var result struct {
		err       error
		win, data *message
		signal    bool
	}

	var wg struct {
		ready  sync.WaitGroup
		finish sync.WaitGroup
	}

	wg.ready.Add(1)  // server signaling readiness to client worker
	wg.finish.Add(2) // server/client signaling test end

	// server loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Done()

		client := server.accept()
		server.handshake(client)

		buf := streambuf.New(nil)
		result.win = server.readMessage(buf, client)
		result.data = server.readMessage(buf, client)
		server.sendACK(client, 1)
		result.err = server.err
	}()

	// worker loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Wait()

		output := makeOutputer()

		signal := outputs.NewSyncSignal()
		output.PublishEvent(signal, time.Now(), testEvent())
		result.signal = signal.Wait()
	}()

	// wait shutdown
	wg.finish.Wait()
	server.Close()

	// validate output
	assert.Nil(t, result.err)
	assert.True(t, result.signal)

	data := result.data
	assert.NotNil(t, result.win)
	assert.NotNil(t, result.data)
	if data != nil {
		assert.Equal(t, 1, len(data.events))
		data = data.events[0]
		assert.Equal(t, 10.0, data.doc["extra"])
		assert.Equal(t, "message", data.doc["message"])
	}

}
Пример #6
0
func (p *syncPublisher) forward(m message) bool {
	sync := outputs.NewSyncSignal()
	m.context.signal = sync
	p.send(m)
	return sync.Wait()
}
Пример #7
0
func (c syncClient) send(m message) bool {
	sync := outputs.NewSyncSignal()
	m.signal = sync
	c(m)
	return sync.Wait()
}