func createEnvelope(eventType events.Envelope_EventType) *events.Envelope {
envelope := &events.Envelope{Origin: proto.String("origin"), EventType: &eventType, Timestamp: proto.Int64(time.Now().UnixNano())}
switch eventType {
case events.Envelope_HttpStartStop:
req, _ := http.NewRequest("GET", "http://www.example.com", nil)
req.RemoteAddr = "www.example.com"
req.Header.Add("User-Agent", "user-agent")
uuid, _ := uuid.NewV4()
envelope.HttpStartStop = factories.NewHttpStartStop(req, http.StatusOK, 128, events.PeerType_Client, uuid)
case events.Envelope_ValueMetric:
envelope.ValueMetric = factories.NewValueMetric("some-value-metric", 123, "km")
case events.Envelope_CounterEvent:
envelope.CounterEvent = factories.NewCounterEvent("some-counter-event", 123)
case events.Envelope_LogMessage:
envelope.LogMessage = factories.NewLogMessage(events.LogMessage_OUT, "some message", "appId", "source")
case events.Envelope_ContainerMetric:
envelope.ContainerMetric = factories.NewContainerMetric("appID", 123, 1, 5, 5)
case events.Envelope_Error:
envelope.Error = factories.NewError("source", 123, "message")
default:
panic(fmt.Sprintf("Unknown event %v\n", eventType))
}
return envelope
}
func (sinkManager *SinkManager) SendSyslogErrorToLoggregator(errorMsg string, appId string) {
sinkManager.logger.Warn(errorMsg)
logMessage := factories.NewLogMessage(events.LogMessage_ERR, errorMsg, appId, "LGR")
envelope, err := emitter.Wrap(logMessage, sinkManager.dropsondeOrigin)
if err != nil {
sinkManager.logger.Warnf("Error marshalling message: %v", err)
return
}
sinkManager.errorChannel <- envelope
}
errorChannel chan *events.Envelope
errorHandler func(string, string, string)
inputChan chan *events.Envelope
bufferSize uint
dialer *net.Dialer
)
BeforeEach(func() {
syslogSinkRunFinished = make(chan bool)
sysLogger = NewSyslogWriterRecorder()
errorChannel = make(chan *events.Envelope, 10)
inputChan = make(chan *events.Envelope)
dialer = &net.Dialer{}
errorHandler = func(errorMsg string, appId string, drainUrl string) {
logMessage := factories.NewLogMessage(events.LogMessage_ERR, errorMsg, appId, "LGR")
envelope, _ := emitter.Wrap(logMessage, "dropsonde-origin")
select {
case errorChannel <- envelope:
default:
}
}
bufferSize = 100
})
JustBeforeEach(func() {
syslogSink = syslog.NewSyslogSink("appId", "syslog://using-fake", loggertesthelper.Logger(), bufferSize, sysLogger, errorHandler, "dropsonde-origin")
})
for i := 0; i < runCount; i++ {
sendLogMessages(fmt.Sprintf("message %d", i), inMessageChan)
}
<-readDone
})
}, 100)
})
Context("lossy", func() {
BeforeEach(func() {
bufferSize = 100
})
var send = func(count int, delay time.Duration) {
msg, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "message", "appId", "App"), "origin")
for i := 0; i < count; i++ {
inMessageChan <- msg
time.Sleep(delay)
}
}
var receive = func(count int, delay time.Duration) (totalLost uint64) {
timeout := time.NewTimer(time.Millisecond)
for i := 0; i < count; i++ {
msgs := buffer.GetOutputChannel()
var msg *events.Envelope
timeout.Reset(time.Millisecond)
select {
case msg = <-msgs:
case <-timeout.C:
Expect(err).NotTo(HaveOccurred())
err = SendAppLog("otherAppId", "message 2", inputConnection)
Expect(err).NotTo(HaveOccurred())
receivedMessageBytes := []byte{}
Eventually(receivedChan).Should(Receive(&receivedMessageBytes))
receivedMessage := DecodeProtoBufLogMessage(receivedMessageBytes)
Expect(receivedMessage.GetAppId()).To(Equal(appID))
Expect(string(receivedMessage.GetMessage())).To(Equal("message 1"))
Expect(receivedChan).To(BeEmpty())
})
It("does not recieve non-log messages", func() {
metricEvent := factories.NewContainerMetric(appID, 0, 10, 10, 10)
SendEvent(metricEvent, inputConnection)
Expect(receivedChan).To(BeEmpty())
})
It("drops invalid log envelopes", func() {
unmarshalledLogMessage := factories.NewLogMessage(events.LogMessage_OUT, "Some Data", appID, "App")
expectedMessage := MarshalEvent(unmarshalledLogMessage, "invalid")
_, err := inputConnection.Write(expectedMessage)
Expect(err).To(BeNil())
Expect(receivedChan).To(BeEmpty())
})
})
"github.com/cloudfoundry/dropsonde/factories"
"github.com/cloudfoundry/sonde-go/events"
"github.com/cloudfoundry/dropsonde/emitter"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Truncating Buffer", func() {
It("works like a channel", func() {
inMessageChan := make(chan *events.Envelope)
buffer := truncatingbuffer.NewTruncatingBuffer(inMessageChan, 2, loggertesthelper.Logger(), "dropsonde-origin")
go buffer.Run()
logMessage1, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "message 1", "appId", "App"), "origin")
inMessageChan <- logMessage1
readMessage := <-buffer.GetOutputChannel()
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1"))
logMessage2, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "message 2", "appId", "App"), "origin")
inMessageChan <- logMessage2
readMessage2 := <-buffer.GetOutputChannel()
Expect(readMessage2.GetLogMessage().GetMessage()).To(ContainSubstring("message 2"))
})
It("works like a truncating channel", func() {
inMessageChan := make(chan *events.Envelope)
buffer := truncatingbuffer.NewTruncatingBuffer(inMessageChan, 2, loggertesthelper.Logger(), "dropsonde-origin")
Expect(websocketSink.ShouldReceiveErrors()).To(BeTrue())
})
})
Describe("Run", func() {
var inputChan chan *events.Envelope
BeforeEach(func() {
inputChan = make(chan *events.Envelope, 10)
})
It("forwards messages", func(done Done) {
defer close(done)
go websocketSink.Run(inputChan)
message, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "hello world", "appId", "App"), "origin")
messageBytes, _ := proto.Marshal(message)
inputChan <- message
Eventually(fakeWebsocket.ReadMessages).Should(HaveLen(1))
Expect(fakeWebsocket.ReadMessages()[0]).To(Equal(messageBytes))
messageTwo, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "goodbye world", "appId", "App"), "origin")
messageTwoBytes, _ := proto.Marshal(messageTwo)
inputChan <- messageTwo
Eventually(fakeWebsocket.ReadMessages).Should(HaveLen(2))
Expect(fakeWebsocket.ReadMessages()[1]).To(Equal(messageTwoBytes))
})
It("sets write deadline", func() {
go websocketSink.Run(inputChan)
})
Describe("failed connections", func() {
It("fails without an appId", func() {
_, connectionDropped = AddWSSink(wsReceivedChan, fmt.Sprintf("ws://%s/apps//stream", apiEndpoint))
Expect(connectionDropped).To(BeClosed())
})
It("fails with bad path", func() {
_, connectionDropped = AddWSSink(wsReceivedChan, fmt.Sprintf("ws://%s/apps/my-app/junk", apiEndpoint))
Expect(connectionDropped).To(BeClosed())
})
})
It("dumps buffer data to the websocket client with /recentlogs", func(done Done) {
lm, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "my message", appId, "App"), "origin")
sinkManager.SendTo(appId, lm)
AddWSSink(wsReceivedChan, fmt.Sprintf("ws://%s/apps/%s/recentlogs", apiEndpoint, appId))
rlm, err := receiveEnvelope(wsReceivedChan)
Expect(err).NotTo(HaveOccurred())
Expect(rlm.GetLogMessage().GetMessage()).To(Equal(lm.GetLogMessage().GetMessage()))
close(done)
})
It("dumps container metric data to the websocket client with /containermetrics", func(done Done) {
cm := factories.NewContainerMetric(appId, 0, 42.42, 1234, 123412341234)
envelope, _ := emitter.Wrap(cm, "origin")
sinkManager.SendTo(appId, envelope)
conn := openTLSConnection(tlsListener.Address())
defer conn.Close()
Expect(tlsListener.Start).Should(Panic())
})
It("panics if you start after a stop", func() {
conn := openTLSConnection(tlsListener.Address())
defer conn.Close()
tlsListener.Stop()
Expect(tlsListener.Start).Should(Panic())
})
It("fails to send message after listener has been stopped", func() {
logMessage := factories.NewLogMessage(events.LogMessage_OUT, "some message", "appId", "source")
envelope, _ := emitter.Wrap(logMessage, "origin")
conn := openTLSConnection(tlsListener.Address())
err := send(conn, envelope)
Expect(err).ToNot(HaveOccurred())
tlsListener.Stop()
Eventually(func() error {
return send(conn, envelope)
}).Should(HaveOccurred())
conn.Close()
})
})
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Streaming Logs", func() {
var inputConnection net.Conn
var appID string
BeforeEach(func() {
guid, _ := uuid.NewV4()
appID = guid.String()
})
itStreams := func(send func(event events.Event, connection net.Conn) error) {
It("receives recent log messages", func() {
logMessage := factories.NewLogMessage(events.LogMessage_OUT, "msg 1", appID, "APP")
err := send(logMessage, inputConnection)
Expect(err).NotTo(HaveOccurred())
returnedMessages := make([][]byte, 1)
Eventually(func() [][]byte {
returnedMessages = retreiveRecentMessages(appID)
return returnedMessages
}).Should(HaveLen(1))
receivedMessage := DecodeProtoBufLogMessage(returnedMessages[0])
Expect(receivedMessage.GetAppId()).To(Equal(appID))
Expect(string(receivedMessage.GetMessage())).To(Equal("msg 1"))
})
go func() {
marshaller.Run(inputChan, outputChan)
close(runComplete)
}()
})
AfterEach(func() {
close(inputChan)
Eventually(runComplete).Should(BeClosed())
})
It("marshals envelopes into bytes", func() {
envelope := &events.Envelope{
Origin: proto.String("fake-origin-1"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "message", "appid", "sourceType"),
}
message, _ := proto.Marshal(envelope)
inputChan <- envelope
outputMessage := <-outputChan
Expect(outputMessage).To(Equal(message))
})
Context("metrics", func() {
var eventuallyExpectCounter = func(name string, value uint64) {
Eventually(func() uint64 { return fakeSender.GetCounter(name) }).Should(BeEquivalentTo(value))
}
It("emits a marshal error counter", func() {
envelope := &events.Envelope{}
Context("Run", func() {
It("runs its collection of unmarshallers in separate go routines", func() {
startingCountGoroutines := runtime.NumGoroutine()
collection.Run(inputChan, outputChan, waitGroup)
Expect(startingCountGoroutines + 5).To(Equal(runtime.NumGoroutine()))
})
})
Context("metrics", func() {
It("emits a total log messages concatenated from the different unmarshallers", func() {
for n := 0; n < 5; n++ {
envelope := &events.Envelope{
Origin: proto.String("fake-origin-3"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message "+string(n), "fake-app-id-1", "DEA"),
}
message, _ := proto.Marshal(envelope)
inputChan <- message
}
collection.Run(inputChan, outputChan, waitGroup)
for n := 0; n < 5; n++ {
<-outputChan
}
metrics := collection.Emit().Metrics
Expect(metrics).NotTo(BeNil())
)
var _ = Describe("Dump Sink", func() {
It("works with one message", func() {
testDump := dump.NewDumpSink("myApp", 1, loggertesthelper.Logger(), time.Second)
dumpRunnerDone := make(chan struct{})
inputChan := make(chan *events.Envelope)
go func() {
testDump.Run(inputChan)
close(dumpRunnerDone)
}()
logMessage, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "hi", "appId", "App"), "origin")
inputChan <- logMessage
close(inputChan)
<-dumpRunnerDone
data := testDump.Dump()
assert.Equal(GinkgoT(), len(data), 1)
Expect(string(data[0].GetLogMessage().GetMessage())).To(Equal("hi"))
})
It("works with two messages", func() {
testDump := dump.NewDumpSink("myApp", 2, loggertesthelper.Logger(), time.Second)
dumpRunnerDone := make(chan struct{})
dialer := &net.Dialer{}
httpsWriter, err := syslogwriter.NewHttpsWriter(url, appId, true, dialer, 0)
Expect(err).ToNot(HaveOccurred())
errorHandler := func(errorMsg string, appId string, drainUrl string) {}
syslogSink := syslog.NewSyslogSink(appId, server.URL, loggertesthelper.Logger(), bufferSize, httpsWriter, errorHandler, "dropsonde-origin")
inputChan := make(chan *events.Envelope)
defer syslogSink.Disconnect()
go syslogSink.Run(inputChan)
for i := 0; i < int(bufferSize); i++ {
msg := fmt.Sprintf("message number %v", i)
logMessage, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, msg, appId, "App"), "origin")
inputChan <- logMessage
}
close(inputChan)
Eventually(func() int64 {
return atomic.LoadInt64(&requests)
}).Should(BeEquivalentTo(bufferSize))
// We ignore the difference in timestamps for the 0th iteration because our exponential backoff
// strategy starts of with a difference of 1 ms
var diff, prevDiff int64
for i := 1; i < len(timestampsInMillis)-1; i++ {
diff = timestampsInMillis[i+1] - timestampsInMillis[i]
Expect(diff).To(BeNumerically(">", 2*prevDiff))
It("emits a value metric counter", func() {
unmarshaller.Write(message)
testhelpers.EventuallyExpectMetric(unmarshaller, "valueMetricReceived", 1)
Eventually(fakeEventEmitter.GetMessages).Should(HaveLen(1))
Expect(fakeEventEmitter.GetMessages()[0].Event.(*events.CounterEvent)).To(Equal(&events.CounterEvent{
Name: proto.String("EventUnmarshaller.valueMetricReceived"),
Delta: proto.Uint64(1),
}))
})
It("emits a total log message counter", func() {
envelope1 := &events.Envelope{
Origin: proto.String("fake-origin-3"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message 1", "fake-app-id-1", "DEA"),
}
envelope2 := &events.Envelope{
Origin: proto.String("fake-origin-3"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "test log message 2", "fake-app-id-2", "DEA"),
}
message1, _ := proto.Marshal(envelope1)
message2, _ := proto.Marshal(envelope2)
unmarshaller.Write(message1)
unmarshaller.Write(message1)
unmarshaller.Write(message2)
func sendLogMessages(message string, inMessageChan chan<- *events.Envelope) {
logMessage1, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, message, "appId", "App"), "origin")
inMessageChan <- logMessage1
}
var _ = Describe("BufferContext", func() {
Context("DefaultContext", func() {
var defaultContext *DefaultContext
BeforeEach(func() {
defaultContext = NewDefaultContext("origin", "testIdentifier")
})
It("Should return a valid properties", func() {
Expect(defaultContext.Origin()).To(Equal("origin"))
Expect(defaultContext.Destination()).To(Equal("testIdentifier"))
for _, event := range events.Envelope_EventType_value {
Expect(defaultContext.EventAllowed(events.Envelope_EventType(event))).To(BeTrue())
}
message := factories.NewLogMessage(events.LogMessage_OUT, "hello", "appID", "source")
envelope := &events.Envelope{
Origin: proto.String("origin"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: message,
}
Expect(defaultContext.AppID(envelope)).To(Equal("appID"))
})
})
Context("LogAllowedContext", func() {
var logAllowedContext *LogAllowedContext
BeforeEach(func() {
logAllowedContext = NewLogAllowedContext("origin", "testIdentifier")
var (
marshaller *eventmarshaller.EventMarshaller
writer *mocks.MockByteArrayWriter
)
BeforeEach(func() {
writer = &mocks.MockByteArrayWriter{}
marshaller = eventmarshaller.New(writer, loggertesthelper.Logger())
})
It("marshals envelopes into bytes", func() {
envelope := &events.Envelope{
Origin: proto.String("fake-origin-1"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "message", "appid", "sourceType"),
}
message, _ := proto.Marshal(envelope)
marshaller.Write(envelope)
Expect(writer.Data()).Should(HaveLen(1))
outputMessage := writer.Data()[0]
Expect(outputMessage).To(Equal(message))
})
Context("metrics", func() {
It("emits the correct metrics context", func() {
Expect(marshaller.Emit().Name).To(Equal("eventMarshaller"))
})
}()
goRoutineSpawned.Wait()
})
AfterEach(func() {
sinkManager.Stop()
TestMessageRouter.Stop()
TestWebsocketServer.Stop()
services.Wait()
})
It("dumps all messages for an app user", func() {
expectedFirstMessageString := "Some data 1"
lm := factories.NewLogMessage(events.LogMessage_OUT, expectedFirstMessageString, "myOtherApp", "APP")
env1, _ := emitter.Wrap(lm, "ORIGIN")
expectedSecondMessageString := "Some data 2"
lm = factories.NewLogMessage(events.LogMessage_OUT, expectedSecondMessageString, "myOtherApp", "APP")
env2, _ := emitter.Wrap(lm, "ORIGIN")
dataReadChannel <- env1
dataReadChannel <- env2
receivedChan := make(chan []byte, 2)
_, stopKeepAlive, droppedChannel := AddWSSink(receivedChan, serverPort, "/apps/myOtherApp/recentlogs")
Eventually(droppedChannel).Should(Receive())
var firstMarshalledEnvelope, secondMarshalledEnvelope []byte
receiveChan = make(chan []byte, 10)
ws, _ = AddWSSink(receiveChan, "4567", "/firehose/hose-subcription-a")
})
AfterEach(func() {
receiveChan = nil
ws.Close()
})
It("listens for dropsonde log message on TLS port", func() {
message := "my-random-tls-message"
guid, err := uuid.NewV4()
Expect(err).NotTo(HaveOccurred())
appID := guid.String()
logMessage := factories.NewLogMessage(events.LogMessage_OUT, message, appID, "APP")
SendEventTCP(logMessage, conn)
receivedMessageBytes := []byte{}
Eventually(receiveChan).Should(Receive(&receivedMessageBytes))
receivedMessage := DecodeProtoBufLogMessage(receivedMessageBytes)
Expect(receivedMessage.GetAppId()).To(Equal(appID))
Expect(string(receivedMessage.GetMessage())).To(Equal(message))
})
It("listens for dropsonde counter event on TLS port", func() {
counterEvent := factories.NewCounterEvent("my-counter", 1)
SendEventTCP(counterEvent, conn)
receivedEventBytes := []byte{}
Describe("SendTo", func() {
It("sends to all known sinks", func() {
sink1 := &channelSink{appId: "myApp",
identifier: "myAppChan1",
done: make(chan struct{}),
}
sink2 := &channelSink{appId: "myApp",
identifier: "myAppChan2",
done: make(chan struct{}),
}
sinkManager.RegisterSink(sink1)
sinkManager.RegisterSink(sink2)
expectedMessageString := "Some Data"
expectedMessage, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, expectedMessageString, "myApp", "App"), "origin")
go sinkManager.SendTo("myApp", expectedMessage)
Eventually(sink1.Received).Should(HaveLen(1))
Eventually(sink2.Received).Should(HaveLen(1))
Expect(sink1.Received()[0]).To(Equal(expectedMessage))
Expect(sink2.Received()[0]).To(Equal(expectedMessage))
})
It("only sends to sinks that match the appID", func(done Done) {
sink1 := &channelSink{appId: "myApp1",
identifier: "myAppChan1",
done: make(chan struct{}),
}
sink2 := &channelSink{appId: "myApp2",
identifier: "myAppChan2",
func (f *fakeSink) UpdateDroppedMessageCount(messageCount int64) {}
var _ = Describe("FirehoseGroup", func() {
It("sends message to all registered sinks", func() {
receiveChan1 := make(chan *events.Envelope, 10)
receiveChan2 := make(chan *events.Envelope, 10)
sink1 := fakeSink{appId: "firehose-a", sinkId: "sink-a"}
sink2 := fakeSink{appId: "firehose-a", sinkId: "sink-b"}
group := firehose_group.NewFirehoseGroup()
group.AddSink(&sink1, receiveChan1)
group.AddSink(&sink2, receiveChan2)
msg, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "test message", "234", "App"), "origin")
group.BroadcastMessage(msg)
var nextChannelToReceive chan *events.Envelope
var rmsg *events.Envelope
select {
case rmsg = <-receiveChan1:
nextChannelToReceive = receiveChan2
case rmsg = <-receiveChan2:
nextChannelToReceive = receiveChan1
}
Expect(rmsg).To(Equal(msg))
group.BroadcastMessage(msg)
Expect(nextChannelToReceive).To(Receive(&msg))
func SendAppLog(appID string, message string, connection net.Conn) error {
logMessage := factories.NewLogMessage(events.LogMessage_OUT, message, appID, "APP")
return SendEvent(logMessage, connection)
}
ContentLength: proto.Int64(3),
}
stopEvent := factories.NewHttpStop(req, 200, 3, events.PeerType_Server, requestId)
Expect(stopEvent.GetTimestamp()).ToNot(BeZero())
stopEvent.Timestamp = nil
Expect(stopEvent).To(Equal(expectedStopEvent))
})
})
Describe("NewLogMessage", func() {
It("should set appropriate fields", func() {
expectedLogEvent := &events.LogMessage{
Message: []byte("hello"),
AppId: proto.String("app-id"),
MessageType: events.LogMessage_OUT.Enum(),
SourceType: proto.String("App"),
}
logEvent := factories.NewLogMessage(events.LogMessage_OUT, "hello", "app-id", "App")
Expect(logEvent.GetTimestamp()).ToNot(BeZero())
logEvent.Timestamp = nil
Expect(logEvent).To(Equal(expectedLogEvent))
})
})
})