func listenForLogs() {
for {
buffer := make([]byte, 1024)
n, _, err := udpConn.ReadFrom(buffer)
if err != nil {
return
}
if n == 0 {
panic("Received empty packet")
}
envelope := new(events.Envelope)
err = proto.Unmarshal(buffer[0:n], envelope)
if err != nil {
panic(err)
}
if envelope.GetEventType() == events.Envelope_LogMessage {
logLock.Lock()
logMessages = append(logMessages, envelope.GetLogMessage())
logLock.Unlock()
}
}
}
func GetAppId(envelope *events.Envelope) string {
if envelope.GetEventType() == events.Envelope_LogMessage {
return envelope.GetLogMessage().GetAppId()
}
if envelope.GetEventType() == events.Envelope_ContainerMetric {
return envelope.GetContainerMetric().GetApplicationId()
}
var event hasAppId
switch envelope.GetEventType() {
case events.Envelope_HttpStart:
event = envelope.GetHttpStart()
case events.Envelope_HttpStop:
event = envelope.GetHttpStop()
case events.Envelope_HttpStartStop:
event = envelope.GetHttpStartStop()
default:
return SystemAppId
}
uuid := event.GetApplicationId()
if uuid != nil {
return formatUUID(uuid)
}
return SystemAppId
}
func TranslateDropsondeToLegacyLogMessage(message []byte) ([]byte, error) {
var receivedEnvelope events.Envelope
err := proto.Unmarshal(message, &receivedEnvelope)
if err != nil {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Unable to unmarshal bytes as Envelope: %v", err)
}
if receivedEnvelope.GetEventType() != events.Envelope_LogMessage {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope contained %s instead of LogMessage", receivedEnvelope.GetEventType().String())
}
logMessage := receivedEnvelope.GetLogMessage()
if logMessage == nil {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope's LogMessage was nil: %v", receivedEnvelope)
}
messageBytes, err := proto.Marshal(
&logmessage.LogMessage{
Message: logMessage.GetMessage(),
MessageType: (*logmessage.LogMessage_MessageType)(logMessage.MessageType),
Timestamp: proto.Int64(logMessage.GetTimestamp()),
AppId: proto.String(logMessage.GetAppId()),
SourceId: proto.String(logMessage.GetSourceInstance()),
SourceName: proto.String(logMessage.GetSourceType()),
},
)
if err != nil {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Failed marshalling converted dropsonde message: %v", err)
}
return messageBytes, nil
}
func (l *logCounter) processEnvelope(env *events.Envelope) {
if env.GetEventType() != events.Envelope_LogMessage {
return
}
logMsg := env.GetLogMessage()
msg := string(logMsg.GetMessage())
if strings.HasPrefix(msg, "mismatched prefix") {
return
}
if !strings.HasPrefix(msg, l.cfg.MessagePrefix) {
fmt.Printf("mismatched prefix: log message %s did not match prefix: %s\n", string(logMsg.GetMessage()), string(l.cfg.MessagePrefix))
return
}
if len(msg) < sepEnd {
fmt.Printf("Cannot parse message %s\n", msg)
return
}
id := Identity{
appID: logMsg.GetAppId(),
runID: msg[prefixEnd:guidEnd],
}
l.counterLock.Lock()
defer l.counterLock.Unlock()
counter, ok := l.counters[id]
if !ok {
counter = make(map[string]bool)
l.counters[id] = counter
}
counter[msg[sepEnd:]] = true
}
func DecodeProtoBufLogMessage(actual []byte) *events.LogMessage {
var receivedEnvelope events.Envelope
err := proto.Unmarshal(actual, &receivedEnvelope)
if err != nil {
Fail(err.Error())
}
return receivedEnvelope.GetLogMessage()
}
func (s *SyslogSink) sendMessage(messageEnvelope *events.Envelope) bool {
logMessage := messageEnvelope.GetLogMessage()
_, err := s.syslogWriter.Write(messagePriorityValue(logMessage), logMessage.GetMessage(), logMessage.GetSourceType(), logMessage.GetSourceInstance(), *logMessage.Timestamp)
if err != nil {
s.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", s.drainUrl, err)
return false
} else {
s.Debugf("Syslog Sink %s: Successfully sent data\n", s.drainUrl)
return true
}
}
func LogMessage(msg *events.Envelope) Event {
logMessage := msg.GetLogMessage()
fields := logrus.Fields{
"cf_app_id": logMessage.GetAppId(),
"timestamp": logMessage.GetTimestamp(),
"source_type": logMessage.GetSourceType(),
"message_type": logMessage.GetMessageType().String(),
"source_instance": logMessage.GetSourceInstance(),
}
return Event{
Fields: fields,
Msg: string(logMessage.GetMessage()),
}
}
func valid(env *events.Envelope) bool {
switch env.GetEventType() {
case events.Envelope_HttpStartStop:
return env.GetHttpStartStop() != nil
case events.Envelope_LogMessage:
return env.GetLogMessage() != nil
case events.Envelope_ValueMetric:
return env.GetValueMetric() != nil
case events.Envelope_CounterEvent:
return env.GetCounterEvent() != nil
case events.Envelope_Error:
return env.GetError() != nil
case events.Envelope_ContainerMetric:
return env.GetContainerMetric() != nil
}
return true
}
client := noaa.NewConsumer(dropsondeEndpoint, &tls.Config{}, nil)
messages := make(chan *events.Envelope)
go client.StreamWithoutReconnect(APP_ID, AUTH_TOKEN, messages)
var request *http.Request
Eventually(fakeDoppler.TrafficControllerConnected, 10).Should(Receive(&request))
Expect(request.URL.Path).To(Equal("/apps/1234/stream"))
currentTime := time.Now().UnixNano()
dropsondeMessage := makeDropsondeMessage("Hello through NOAA", APP_ID, currentTime)
fakeDoppler.SendLogMessage(dropsondeMessage)
var receivedEnvelope *events.Envelope
Eventually(messages).Should(Receive(&receivedEnvelope))
receivedMessage := receivedEnvelope.GetLogMessage()
Expect(receivedMessage.GetMessage()).To(BeEquivalentTo("Hello through NOAA"))
Expect(receivedMessage.GetAppId()).To(Equal(APP_ID))
Expect(receivedMessage.GetTimestamp()).To(Equal(currentTime))
client.Close()
})
It("closes the upstream websocket connection when done", func() {
client := noaa.NewConsumer(dropsondeEndpoint, &tls.Config{}, nil)
messages := make(chan *events.Envelope)
go client.StreamWithoutReconnect(APP_ID, AUTH_TOKEN, messages)
var request *http.Request
Eventually(fakeDoppler.TrafficControllerConnected, 10).Should(Receive(&request))
Eventually(fakeDoppler.ConnectionPresent).Should(BeTrue())
receivedChan := make(chan []byte, 2)
_, stopKeepAlive, droppedChannel := AddWSSink(receivedChan, serverPort, "/apps/myOtherApp/recentlogs")
Eventually(droppedChannel).Should(Receive())
var firstMarshalledEnvelope, secondMarshalledEnvelope []byte
Eventually(receivedChan).Should(Receive(&firstMarshalledEnvelope))
Eventually(receivedChan).Should(Receive(&secondMarshalledEnvelope))
var envelope1 events.Envelope
var envelope2 events.Envelope
proto.Unmarshal(firstMarshalledEnvelope, &envelope1)
proto.Unmarshal(secondMarshalledEnvelope, &envelope2)
Expect(envelope1.GetLogMessage().GetMessage()).To(BeEquivalentTo(expectedFirstMessageString))
Expect(envelope2.GetLogMessage().GetMessage()).To(BeEquivalentTo(expectedSecondMessageString))
stopKeepAlive <- true
})
It("doesn't hang when there are no messages", func() {
receivedChan := make(chan []byte, 1)
AddWSSink(receivedChan, serverPort, "/apps/myOtherApp/recentlogs")
doneChan := make(chan bool)
go func() {
dumpAllMessages(receivedChan)
close(doneChan)
}()
})
})
Context("as a channel", func() {
JustBeforeEach(func() {
go buffer.Run()
})
It("works like a channel", func() {
sendLogMessages("message 1", inMessageChan)
var readMessage *events.Envelope
Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage))
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1"))
sendLogMessages("message 2", inMessageChan)
Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage))
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 2"))
})
Context("tracking dropped messages", func() {
var fakeEventEmitter *fake.FakeEventEmitter
receiveDroppedMessages := func() uint64 {
var t uint64
for i, _ := range fakeEventEmitter.GetMessages() {
e := fakeEventEmitter.GetMessages()[i].Event.(*events.CounterEvent)
})
JustBeforeEach(func() {
incoming, errors = cnsmr.StreamWithoutReconnect(appGuid, authToken)
})
Context("when there is no TLS Config or consumerProxyFunc setting", func() {
Context("when the connection can be established", func() {
It("receives messages on the incoming channel", func(done Done) {
defer close(done)
fakeHandler.InputChan <- marshalMessage(createMessage("hello", 0))
var message *events.Envelope
Eventually(incoming).Should(Receive(&message))
Expect(message.GetLogMessage().GetMessage()).To(Equal([]byte("hello")))
fakeHandler.Close()
})
Context("with a specific app ID", func() {
BeforeEach(func() {
appGuid = "the-app-guid"
})
It("sends messages for a specific app", func() {
fakeHandler.Close()
Eventually(fakeHandler.GetLastURL).Should(ContainSubstring("/apps/the-app-guid/stream"))
})
})
})
})
Context("as a channel", func() {
JustBeforeEach(func() {
go buffer.Run()
})
It("works like a channel", func() {
sendLogMessages("message 1", inMessageChan)
var readMessage *events.Envelope
Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage))
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1"))
sendLogMessages("message 2", inMessageChan)
Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage))
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 2"))
})
Context("tracking dropped messages", func() {
var (
fakeEventEmitter *fake.FakeEventEmitter
mockBatcher *mockMetricBatcher
)
tracksDroppedMessagesAnd := func(itMsg string, delta, total int) {
})
})
Context("as a channel", func() {
JustBeforeEach(func() {
go buffer.Run()
})
It("works like a channel", func() {
sendLogMessages("message 1", inMessageChan)
var readMessage *events.Envelope
Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage))
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1"))
sendLogMessages("message 2", inMessageChan)
Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage))
Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 2"))
})
Context("tracking dropped messages", func() {
var fakeEventEmitter *fake.FakeEventEmitter
BeforeEach(func() {
fakeEventEmitter = fake.NewFakeEventEmitter("doppler")
sender := metric_sender.NewMetricSender(fakeEventEmitter)
batcher := metricbatcher.New(sender, 100*time.Millisecond)
lm, _ = emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "my message", appId, "App"), "origin")
})
AfterEach(func() {
close(stopKeepAlive)
<-connectionDropped
cleanup()
})
It("sends data to the websocket firehose client", func() {
sinkManager.SendTo(appId, lm)
rlm, err := receiveEnvelope(wsReceivedChan)
Expect(err).NotTo(HaveOccurred())
Expect(rlm.GetLogMessage().GetMessage()).To(Equal(lm.GetLogMessage().GetMessage()))
})
Context("when data is sent to the websocket firehose client", func() {
JustBeforeEach(func() {
sinkManager.SendTo(appId, lm)
})
It("emits counter metric sentMessagesFirehose", func() {
Eventually(mockBatcher.BatchCounterInput).Should(BeCalled(
With("sentMessagesFirehose"),
))
Eventually(mockChainer.SetTagInput).Should(BeCalled(
With("subscription_id", subscriptionID),
))
bytes := generator.Generate()
var envelope events.Envelope
err := proto.Unmarshal(bytes, &envelope)
Expect(err).ToNot(HaveOccurred())
Expect(envelope.GetEventType()).To(Equal(events.Envelope_ValueMetric))
})
})
Context("The LogGenerator", func() {
It("generates a log message envelope", func() {
generator := messagegenerator.NewLogMessageGenerator()
bytes := generator.Generate()
var envelope events.Envelope
err := proto.Unmarshal(bytes, &envelope)
Expect(err).ToNot(HaveOccurred())
Expect(envelope.GetLogMessage()).ToNot(BeNil())
Expect(envelope.GetLogMessage().GetMessage()).To(BeEquivalentTo("test message"))
})
})
Context("The LegacyLogGenerator", func() {
It("generates a legacy log message envelope", func() {
generator := messagegenerator.NewLegacyLogGenerator()
bytes := generator.Generate()
var envelope logmessage.LogEnvelope
err := proto.Unmarshal(bytes, &envelope)
Expect(err).ToNot(HaveOccurred())
Expect(envelope.GetLogMessage()).ToNot(BeNil())
Expect(envelope.GetLogMessage().GetMessage()).To(BeEquivalentTo("test message"))
})
})
