func (o *outputWorker) onBulk(ctx *context, events []common.MapStr) {
if len(events) == 0 {
debug("output worker: no events to publish")
outputs.SignalCompleted(ctx.signal)
return
}
var sync *outputs.SyncSignal
if ctx.sync {
sync = outputs.NewSyncSignal()
}
if o.maxBulkSize < 0 || len(events) <= o.maxBulkSize {
o.sendBulk(sync, ctx, events)
return
}
// start splitting bulk request
splits := (len(events) + (o.maxBulkSize - 1)) / o.maxBulkSize
ctx.signal = outputs.NewSplitSignaler(ctx.signal, splits)
for len(events) > 0 {
sz := o.maxBulkSize
if sz > len(events) {
sz = len(events)
}
o.sendBulk(sync, ctx, events[:sz])
events = events[sz:]
}
}
func testSendMultipleBatchesViaLogstash(
t *testing.T,
name string,
numBatches int,
batchSize int,
tls bool,
) {
if testing.Short() {
t.Skip("Skipping in short mode. Requires Logstash and Elasticsearch")
}
ls := newTestLogstashOutput(t, name, tls)
defer ls.Cleanup()
batches := make([][]common.MapStr, 0, numBatches)
for i := 0; i < numBatches; i++ {
batch := make([]common.MapStr, 0, batchSize)
for j := 0; j < batchSize; j++ {
event := common.MapStr{
"@timestamp": common.Time(time.Now()),
"host": "test-host",
"type": "log",
"message": fmt.Sprintf("batch hello world - %v", i*batchSize+j),
}
batch = append(batch, event)
}
batches = append(batches, batch)
}
for _, batch := range batches {
sig := outputs.NewSyncSignal()
ls.BulkPublish(sig, time.Now(), batch)
ok := sig.Wait()
assert.Equal(t, true, ok)
}
// wait for logstash event flush + elasticsearch
ok := waitUntilTrue(5*time.Second, checkIndex(ls, numBatches*batchSize))
assert.True(t, ok) // check number of events matches total number of events
// search value in logstash elasticsearch index
resp, err := ls.Read()
if err != nil {
return
}
if len(resp) != 10 {
t.Errorf("wrong number of results: %d", len(resp))
}
}
func (o *outputWorker) onEvent(ctx *context, event common.MapStr) {
debug("output worker: publish single event")
ts := time.Time(event["@timestamp"].(common.Time)).UTC()
if !ctx.sync {
_ = o.out.PublishEvent(ctx.signal, ts, event)
return
}
signal := outputs.NewSyncSignal()
for {
o.out.PublishEvent(signal, ts, event)
if signal.Wait() {
outputs.SignalCompleted(ctx.signal)
break
}
}
}
func TestLogstashInvalidTLS(t *testing.T) {
certName := "ca_invalid_test"
ip := net.IP{1, 2, 3, 4}
timeout := 2 * time.Second
genCertsForIPIfMIssing(t, ip, certName)
server := newMockTLSServer(t, timeout, certName)
retries := 0
config := outputs.MothershipConfig{
TLS: &outputs.TLSConfig{
CAs: []string{certName + ".pem"},
},
Timeout: 1,
MaxRetries: &retries,
Hosts: []string{server.Addr()},
}
var result struct {
err error
handshakeFail bool
signal bool
}
var wg struct {
ready sync.WaitGroup
finish sync.WaitGroup
}
wg.ready.Add(1) // server signaling readiness to client worker
wg.finish.Add(2) // server/client signaling test end
// server loop
go func() {
defer wg.finish.Done()
wg.ready.Done()
client := server.accept()
if server.err != nil {
t.Fatalf("server error: %v", server.err)
}
server.handshake(client)
result.handshakeFail = server.err != nil
}()
// client loop
go func() {
defer wg.finish.Done()
wg.ready.Wait()
output := newTestLumberjackOutput(t, "", &config)
signal := outputs.NewSyncSignal()
output.PublishEvent(signal, time.Now(), testEvent())
result.signal = signal.Wait()
}()
// wait shutdown
wg.finish.Wait()
server.Close()
// validate output
assert.True(t, result.handshakeFail)
assert.False(t, result.signal)
}
func testConnectionType(
t *testing.T,
server *mockLSServer,
makeOutputer func() outputs.BulkOutputer,
) {
var result struct {
err error
win, data *message
signal bool
}
var wg struct {
ready sync.WaitGroup
finish sync.WaitGroup
}
wg.ready.Add(1) // server signaling readiness to client worker
wg.finish.Add(2) // server/client signaling test end
// server loop
go func() {
defer wg.finish.Done()
wg.ready.Done()
client := server.accept()
server.handshake(client)
buf := streambuf.New(nil)
result.win = server.readMessage(buf, client)
result.data = server.readMessage(buf, client)
server.sendACK(client, 1)
result.err = server.err
}()
// worker loop
go func() {
defer wg.finish.Done()
wg.ready.Wait()
output := makeOutputer()
signal := outputs.NewSyncSignal()
output.PublishEvent(signal, time.Now(), testEvent())
result.signal = signal.Wait()
}()
// wait shutdown
wg.finish.Wait()
server.Close()
// validate output
assert.Nil(t, result.err)
assert.True(t, result.signal)
data := result.data
assert.NotNil(t, result.win)
assert.NotNil(t, result.data)
if data != nil {
assert.Equal(t, 1, len(data.events))
data = data.events[0]
assert.Equal(t, 10.0, data.doc["extra"])
assert.Equal(t, "message", data.doc["message"])
}
}
func (p *syncPublisher) forward(m message) bool {
sync := outputs.NewSyncSignal()
m.context.signal = sync
p.send(m)
return sync.Wait()
}
func (c syncClient) send(m message) bool {
sync := outputs.NewSyncSignal()
m.signal = sync
c(m)
return sync.Wait()
}