func makeFixedMessage(encoder client.Encoder, size uint64) [][]byte {
ma := make([][]byte, 1)
hostname, _ := os.Hostname()
pid := int32(os.Getpid())
msg := &message.Message{}
msg.SetType("hekabench")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetSeverity(int32(6))
msg.SetEnvVersion("0.8")
msg.SetPid(pid)
msg.SetHostname(hostname)
rdm := &randomDataMaker{
src: rand.NewSource(time.Now().UnixNano()),
}
buf := make([]byte, size)
payloadSuffix := bytes.NewBuffer(buf)
_, err := io.CopyN(payloadSuffix, rdm, int64(size))
payload := fmt.Sprintf("hekabench: %s", hostname)
if err == nil {
payload = fmt.Sprintf("%s - %s", payload, payloadSuffix.String())
} else {
log.Println("Error getting random string: ", err)
}
msg.SetPayload(payload)
var stream []byte
if err := encoder.EncodeMessageStream(msg, &stream); err != nil {
log.Println(err)
}
ma[0] = stream
return ma
}
func makeVariableMessage(encoder client.Encoder, items int, rdm *randomDataMaker) [][]byte {
ma := make([][]byte, items)
hostname, _ := os.Hostname()
pid := int32(os.Getpid())
var cnt int
for x := 0; x < items; x++ {
msg := &message.Message{}
msg.SetUuid(uuid.NewRandom())
msg.SetTimestamp(time.Now().UnixNano())
msg.SetType("hekabench")
msg.SetLogger("flood")
msg.SetSeverity(int32(0))
msg.SetEnvVersion("0.2")
msg.SetPid(pid)
msg.SetHostname(hostname)
cnt = (rand.Int() % 3) * 1024
msg.SetPayload(makePayload(uint64(cnt), rdm))
cnt = rand.Int() % 5
for c := 0; c < cnt; c++ {
field, _ := message.NewField(fmt.Sprintf("string%d", c), fmt.Sprintf("value%d", c), "")
msg.AddField(field)
}
cnt = rand.Int() % 5
for c := 0; c < cnt; c++ {
b := byte(c)
field, _ := message.NewField(fmt.Sprintf("bytes%d", c), []byte{b, b, b, b, b, b, b, b}, "")
msg.AddField(field)
}
cnt = rand.Int() % 5
for c := 0; c < cnt; c++ {
field, _ := message.NewField(fmt.Sprintf("int%d", c), c, "")
msg.AddField(field)
}
cnt = rand.Int() % 5
for c := 0; c < cnt; c++ {
field, _ := message.NewField(fmt.Sprintf("double%d", c), float64(c), "")
msg.AddField(field)
}
cnt = rand.Int() % 5
for c := 0; c < cnt; c++ {
field, _ := message.NewField(fmt.Sprintf("bool%d", c), true, "")
msg.AddField(field)
}
cnt = (rand.Int() % 60) * 1024
buf := make([]byte, cnt)
field, _ := message.NewField("filler", buf, "")
msg.AddField(field)
var stream []byte
if err := encoder.EncodeMessageStream(msg, &stream); err != nil {
log.Println(err)
}
ma[x] = stream
}
return ma
}
func makeFixedMessage(encoder client.Encoder, size uint64, rdm *randomDataMaker) [][]byte {
ma := make([][]byte, 1)
hostname, _ := os.Hostname()
pid := int32(os.Getpid())
msg := &message.Message{}
msg.SetType("hekabench")
msg.SetTimestamp(time.Now().UnixNano())
msg.SetUuid(uuid.NewRandom())
msg.SetSeverity(int32(6))
msg.SetEnvVersion("0.8")
msg.SetPid(pid)
msg.SetHostname(hostname)
msg.SetPayload(makePayload(size, rdm))
var stream []byte
if err := encoder.EncodeMessageStream(msg, &stream); err != nil {
log.Println(err)
}
ma[0] = stream
return ma
}
func (no *NsqOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var (
encoder client.Encoder
msg *message.Message
msgBody []byte = make([]byte, 0, 1024)
pack *pipeline.PipelinePack
)
conf := no.conf
encoder = client.NewProtobufEncoder(nil)
for pack = range or.InChan() {
if conf.Serialize {
msg = pack.Message
if err = encoder.EncodeMessageStream(msg, &msgBody); err != nil {
or.LogError(err)
err = nil
pack.Recycle()
continue
}
//err := no.nsqwriter.PublishAsync(conf.Topic, []byte(pack.Message.GetPayload()), nil)
//err = no.nsqwriter.PublishAsync(conf.Topic, msgBody, nil)
_, _, err = no.nsqwriter.Publish(conf.Topic, msgBody)
if err != nil {
or.LogError(fmt.Errorf("error in writer.PublishAsync"))
}
msgBody = msgBody[:0]
} else {
err = no.nsqwriter.PublishAsync(conf.Topic, []byte(pack.Message.GetPayload()), nil)
if err != nil {
or.LogError(fmt.Errorf("error in writer.PublishAsync"))
}
}
pack.Recycle()
}
return
}
func (ao *AMQPOutput) Run(or OutputRunner, h PluginHelper) (err error) {
inChan := or.InChan()
conf := ao.config
var (
pack *PipelinePack
msg *message.Message
persist uint8
ok bool = true
amqpMsg amqp.Publishing
encoder client.Encoder
msgBody []byte = make([]byte, 0, 500)
)
if conf.Persistent {
persist = uint8(1)
} else {
persist = uint8(0)
}
encoder = client.NewProtobufEncoder(nil)
for ok {
select {
case <-ao.closeChan:
ok = false
case pack, ok = <-inChan:
if !ok {
break
}
msg = pack.Message
if conf.Serialize {
if err = encoder.EncodeMessageStream(msg, &msgBody); err != nil {
or.LogError(err)
err = nil
pack.Recycle()
continue
}
amqpMsg = amqp.Publishing{
DeliveryMode: persist,
Timestamp: time.Now(),
ContentType: "application/hekad",
Body: msgBody,
}
} else {
amqpMsg = amqp.Publishing{
DeliveryMode: persist,
Timestamp: time.Now(),
ContentType: "text/plain",
Body: []byte(msg.GetPayload()),
}
}
err = ao.ch.Publish(conf.Exchange, conf.RoutingKey,
false, false, amqpMsg)
if err != nil {
ok = false
} else {
pack.Recycle()
}
msgBody = msgBody[:0]
}
}
ao.usageWg.Done()
amqpHub.Close(conf.URL, ao.connWg)
ao.connWg.Wait()
return
}
