Esempio n. 1
0
func (p *syncPipeline) publish(m message) bool {
	if p.pub.disabled {
		debug("publisher disabled")
		op.SigCompleted(m.context.Signal)
		return true
	}

	client := m.client
	signal := m.context.Signal
	sync := op.NewSignalChannel()
	if len(p.pub.Output) > 1 {
		m.context.Signal = op.SplitSignaler(sync, len(p.pub.Output))
	} else {
		m.context.Signal = sync
	}

	for _, o := range p.pub.Output {
		o.send(m)
	}

	// Await completion signal from output plugin. If client has been disconnected
	// ignore any signal and drop events no matter if send or not.
	select {
	case <-client.canceler.Done():
		return false // return false, indicating events potentially not being send
	case sig := <-sync.C:
		sig.Apply(signal)
		return sig == op.SignalCompleted
	}
}
func testSendMultipleBatchesViaLogstash(
	t *testing.T,
	name string,
	numBatches int,
	batchSize int,
	tls bool,
) {

	ls := newTestLogstashOutput(t, name, tls)
	defer ls.Cleanup()

	batches := make([][]common.MapStr, 0, numBatches)
	for i := 0; i < numBatches; i++ {
		batch := make([]common.MapStr, 0, batchSize)
		for j := 0; j < batchSize; j++ {
			event := common.MapStr{
				"@timestamp": common.Time(time.Now()),
				"host":       "test-host",
				"type":       "log",
				"message":    fmt.Sprintf("batch hello world - %v", i*batchSize+j),
			}
			batch = append(batch, event)
		}
		batches = append(batches, batch)
	}

	for _, batch := range batches {
		sig := op.NewSignalChannel()
		ls.BulkPublish(sig, testOptions, batch)
		ok := sig.Wait() == op.SignalCompleted
		assert.Equal(t, true, ok)
	}

	// wait for logstash event flush + elasticsearch
	ok := waitUntilTrue(5*time.Second, checkIndex(ls, numBatches*batchSize))
	assert.True(t, ok) // check number of events matches total number of events

	// search value in logstash elasticsearch index
	resp, err := ls.Read()
	if err != nil {
		return
	}
	if len(resp) != 10 {
		t.Errorf("wrong number of results: %d", len(resp))
	}
}
Esempio n. 3
0
func testConnectionType(
	t *testing.T,
	mock *transptest.MockServer,
	makeOutputer func() outputs.BulkOutputer,
) {
	t.Log("testConnectionType")
	server, _ := v2.NewWithListener(mock.Listener)

	// worker loop
	go func() {
		t.Log("start worker loop")
		defer t.Log("stop worker loop")

		t.Log("make outputter")
		output := makeOutputer()
		t.Logf("new outputter: %v", output)

		signal := op.NewSignalChannel()
		t.Log("publish event")
		output.PublishEvent(signal, testOptions, testEvent())

		t.Log("wait signal")
		assert.True(t, signal.Wait() == op.SignalCompleted)

		server.Close()
	}()

	for batch := range server.ReceiveChan() {
		batch.ACK()

		events := batch.Events
		assert.Equal(t, 1, len(events))
		msg := events[0].(map[string]interface{})
		assert.Equal(t, 10.0, msg["extra"])
		assert.Equal(t, "message", msg["message"])
	}
}
Esempio n. 4
0
func TestLogstashInvalidTLS(t *testing.T) {
	certName := "ca_invalid_test"
	ip := net.IP{1, 2, 3, 4}

	timeout := 2 * time.Second
	transptest.GenCertsForIPIfMIssing(t, ip, certName)
	server := newMockTLSServer(t, timeout, certName)

	config := map[string]interface{}{
		"hosts":                       []string{server.Addr()},
		"index":                       testLogstashIndex("logstash-tls-invalid"),
		"timeout":                     1,
		"max_retries":                 0,
		"tls.certificate_authorities": []string{certName + ".pem"},
	}

	var result struct {
		err           error
		handshakeFail bool
		signal        bool
	}

	var wg struct {
		ready  sync.WaitGroup
		finish sync.WaitGroup
	}

	wg.ready.Add(1)  // server signaling readiness to client worker
	wg.finish.Add(2) // server/client signaling test end

	// server loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Done()

		client := server.Accept()
		if server.Err != nil {
			t.Fatalf("server error: %v", server.Err)
		}

		server.Handshake(client)
		result.handshakeFail = server.Err != nil
	}()

	// client loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Wait()

		output := newTestLumberjackOutput(t, "", config)

		signal := op.NewSignalChannel()
		output.PublishEvent(signal, testOptions, testEvent())
		result.signal = signal.Wait() == op.SignalCompleted
	}()

	// wait shutdown
	wg.finish.Wait()
	server.Close()

	// validate output
	assert.True(t, result.handshakeFail)
	assert.False(t, result.signal)
}
Esempio n. 5
0
func testConnectionType(
	t *testing.T,
	server *mockLSServer,
	makeOutputer func() outputs.BulkOutputer,
) {
	var result struct {
		err       error
		win, data *message
		signal    bool
	}

	var wg struct {
		ready  sync.WaitGroup
		finish sync.WaitGroup
	}

	t.Log("testConnectionType")

	wg.ready.Add(1)  // server signaling readiness to client worker
	wg.finish.Add(2) // server/client signaling test end

	// server loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Done()

		t.Log("start server loop")
		defer t.Log("stop server loop")

		client := server.Accept()
		server.Handshake(client)

		buf := streambuf.New(nil)
		result.win = server.readMessage(buf, client)
		result.data = server.readMessage(buf, client)
		server.sendACK(client, 1)
		result.err = server.Err
	}()

	// worker loop
	go func() {
		defer wg.finish.Done()
		wg.ready.Wait()

		t.Log("start worker loop")
		defer t.Log("stop worker loop")

		t.Log("make outputter")
		output := makeOutputer()
		t.Logf("new outputter: %v", output)

		signal := op.NewSignalChannel()
		t.Log("publish event")
		output.PublishEvent(signal, testOptions, testEvent())

		t.Log("wait signal")
		result.signal = signal.Wait() == op.SignalCompleted
	}()

	// wait shutdown
	wg.finish.Wait()
	server.Close()

	// validate output
	assert.Nil(t, result.err)
	assert.True(t, result.signal)

	data := result.data
	assert.NotNil(t, result.win)
	assert.NotNil(t, result.data)
	if data != nil {
		assert.Equal(t, 1, len(data.events))
		data = data.events[0]
		assert.Equal(t, 10.0, data.doc["extra"])
		assert.Equal(t, "message", data.doc["message"])
	}

}