func listenForLogs() { for { buffer := make([]byte, 1024) n, _, err := udpConn.ReadFrom(buffer) if err != nil { return } if n == 0 { panic("Received empty packet") } envelope := new(events.Envelope) err = proto.Unmarshal(buffer[0:n], envelope) if err != nil { panic(err) } if envelope.GetEventType() == events.Envelope_LogMessage { logLock.Lock() logMessages = append(logMessages, envelope.GetLogMessage()) logLock.Unlock() } } }
func GetAppId(envelope *events.Envelope) string { if envelope.GetEventType() == events.Envelope_LogMessage { return envelope.GetLogMessage().GetAppId() } if envelope.GetEventType() == events.Envelope_ContainerMetric { return envelope.GetContainerMetric().GetApplicationId() } var event hasAppId switch envelope.GetEventType() { case events.Envelope_HttpStart: event = envelope.GetHttpStart() case events.Envelope_HttpStop: event = envelope.GetHttpStop() case events.Envelope_HttpStartStop: event = envelope.GetHttpStartStop() default: return SystemAppId } uuid := event.GetApplicationId() if uuid != nil { return formatUUID(uuid) } return SystemAppId }
func TranslateDropsondeToLegacyLogMessage(message []byte) ([]byte, error) { var receivedEnvelope events.Envelope err := proto.Unmarshal(message, &receivedEnvelope) if err != nil { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Unable to unmarshal bytes as Envelope: %v", err) } if receivedEnvelope.GetEventType() != events.Envelope_LogMessage { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope contained %s instead of LogMessage", receivedEnvelope.GetEventType().String()) } logMessage := receivedEnvelope.GetLogMessage() if logMessage == nil { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope's LogMessage was nil: %v", receivedEnvelope) } messageBytes, err := proto.Marshal( &logmessage.LogMessage{ Message: logMessage.GetMessage(), MessageType: (*logmessage.LogMessage_MessageType)(logMessage.MessageType), Timestamp: proto.Int64(logMessage.GetTimestamp()), AppId: proto.String(logMessage.GetAppId()), SourceId: proto.String(logMessage.GetSourceInstance()), SourceName: proto.String(logMessage.GetSourceType()), }, ) if err != nil { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Failed marshalling converted dropsonde message: %v", err) } return messageBytes, nil }
func (l *logCounter) processEnvelope(env *events.Envelope) { if env.GetEventType() != events.Envelope_LogMessage { return } logMsg := env.GetLogMessage() msg := string(logMsg.GetMessage()) if strings.HasPrefix(msg, "mismatched prefix") { return } if !strings.HasPrefix(msg, l.cfg.MessagePrefix) { fmt.Printf("mismatched prefix: log message %s did not match prefix: %s\n", string(logMsg.GetMessage()), string(l.cfg.MessagePrefix)) return } if len(msg) < sepEnd { fmt.Printf("Cannot parse message %s\n", msg) return } id := Identity{ appID: logMsg.GetAppId(), runID: msg[prefixEnd:guidEnd], } l.counterLock.Lock() defer l.counterLock.Unlock() counter, ok := l.counters[id] if !ok { counter = make(map[string]bool) l.counters[id] = counter } counter[msg[sepEnd:]] = true }
func DecodeProtoBufLogMessage(actual []byte) *events.LogMessage { var receivedEnvelope events.Envelope err := proto.Unmarshal(actual, &receivedEnvelope) if err != nil { Fail(err.Error()) } return receivedEnvelope.GetLogMessage() }
func (s *SyslogSink) sendMessage(messageEnvelope *events.Envelope) bool { logMessage := messageEnvelope.GetLogMessage() _, err := s.syslogWriter.Write(messagePriorityValue(logMessage), logMessage.GetMessage(), logMessage.GetSourceType(), logMessage.GetSourceInstance(), *logMessage.Timestamp) if err != nil { s.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", s.drainUrl, err) return false } else { s.Debugf("Syslog Sink %s: Successfully sent data\n", s.drainUrl) return true } }
func LogMessage(msg *events.Envelope) Event { logMessage := msg.GetLogMessage() fields := logrus.Fields{ "cf_app_id": logMessage.GetAppId(), "timestamp": logMessage.GetTimestamp(), "source_type": logMessage.GetSourceType(), "message_type": logMessage.GetMessageType().String(), "source_instance": logMessage.GetSourceInstance(), } return Event{ Fields: fields, Msg: string(logMessage.GetMessage()), } }
func valid(env *events.Envelope) bool { switch env.GetEventType() { case events.Envelope_HttpStartStop: return env.GetHttpStartStop() != nil case events.Envelope_LogMessage: return env.GetLogMessage() != nil case events.Envelope_ValueMetric: return env.GetValueMetric() != nil case events.Envelope_CounterEvent: return env.GetCounterEvent() != nil case events.Envelope_Error: return env.GetError() != nil case events.Envelope_ContainerMetric: return env.GetContainerMetric() != nil } return true }
client := noaa.NewConsumer(dropsondeEndpoint, &tls.Config{}, nil) messages := make(chan *events.Envelope) go client.StreamWithoutReconnect(APP_ID, AUTH_TOKEN, messages) var request *http.Request Eventually(fakeDoppler.TrafficControllerConnected, 10).Should(Receive(&request)) Expect(request.URL.Path).To(Equal("/apps/1234/stream")) currentTime := time.Now().UnixNano() dropsondeMessage := makeDropsondeMessage("Hello through NOAA", APP_ID, currentTime) fakeDoppler.SendLogMessage(dropsondeMessage) var receivedEnvelope *events.Envelope Eventually(messages).Should(Receive(&receivedEnvelope)) receivedMessage := receivedEnvelope.GetLogMessage() Expect(receivedMessage.GetMessage()).To(BeEquivalentTo("Hello through NOAA")) Expect(receivedMessage.GetAppId()).To(Equal(APP_ID)) Expect(receivedMessage.GetTimestamp()).To(Equal(currentTime)) client.Close() }) It("closes the upstream websocket connection when done", func() { client := noaa.NewConsumer(dropsondeEndpoint, &tls.Config{}, nil) messages := make(chan *events.Envelope) go client.StreamWithoutReconnect(APP_ID, AUTH_TOKEN, messages) var request *http.Request Eventually(fakeDoppler.TrafficControllerConnected, 10).Should(Receive(&request)) Eventually(fakeDoppler.ConnectionPresent).Should(BeTrue())
receivedChan := make(chan []byte, 2) _, stopKeepAlive, droppedChannel := AddWSSink(receivedChan, serverPort, "/apps/myOtherApp/recentlogs") Eventually(droppedChannel).Should(Receive()) var firstMarshalledEnvelope, secondMarshalledEnvelope []byte Eventually(receivedChan).Should(Receive(&firstMarshalledEnvelope)) Eventually(receivedChan).Should(Receive(&secondMarshalledEnvelope)) var envelope1 events.Envelope var envelope2 events.Envelope proto.Unmarshal(firstMarshalledEnvelope, &envelope1) proto.Unmarshal(secondMarshalledEnvelope, &envelope2) Expect(envelope1.GetLogMessage().GetMessage()).To(BeEquivalentTo(expectedFirstMessageString)) Expect(envelope2.GetLogMessage().GetMessage()).To(BeEquivalentTo(expectedSecondMessageString)) stopKeepAlive <- true }) It("doesn't hang when there are no messages", func() { receivedChan := make(chan []byte, 1) AddWSSink(receivedChan, serverPort, "/apps/myOtherApp/recentlogs") doneChan := make(chan bool) go func() { dumpAllMessages(receivedChan) close(doneChan) }()
}) }) Context("as a channel", func() { JustBeforeEach(func() { go buffer.Run() }) It("works like a channel", func() { sendLogMessages("message 1", inMessageChan) var readMessage *events.Envelope Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage)) Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1")) sendLogMessages("message 2", inMessageChan) Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage)) Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 2")) }) Context("tracking dropped messages", func() { var fakeEventEmitter *fake.FakeEventEmitter receiveDroppedMessages := func() uint64 { var t uint64 for i, _ := range fakeEventEmitter.GetMessages() { e := fakeEventEmitter.GetMessages()[i].Event.(*events.CounterEvent)
}) JustBeforeEach(func() { incoming, errors = cnsmr.StreamWithoutReconnect(appGuid, authToken) }) Context("when there is no TLS Config or consumerProxyFunc setting", func() { Context("when the connection can be established", func() { It("receives messages on the incoming channel", func(done Done) { defer close(done) fakeHandler.InputChan <- marshalMessage(createMessage("hello", 0)) var message *events.Envelope Eventually(incoming).Should(Receive(&message)) Expect(message.GetLogMessage().GetMessage()).To(Equal([]byte("hello"))) fakeHandler.Close() }) Context("with a specific app ID", func() { BeforeEach(func() { appGuid = "the-app-guid" }) It("sends messages for a specific app", func() { fakeHandler.Close() Eventually(fakeHandler.GetLastURL).Should(ContainSubstring("/apps/the-app-guid/stream")) }) })
}) }) Context("as a channel", func() { JustBeforeEach(func() { go buffer.Run() }) It("works like a channel", func() { sendLogMessages("message 1", inMessageChan) var readMessage *events.Envelope Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage)) Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1")) sendLogMessages("message 2", inMessageChan) Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage)) Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 2")) }) Context("tracking dropped messages", func() { var ( fakeEventEmitter *fake.FakeEventEmitter mockBatcher *mockMetricBatcher ) tracksDroppedMessagesAnd := func(itMsg string, delta, total int) {
}) }) Context("as a channel", func() { JustBeforeEach(func() { go buffer.Run() }) It("works like a channel", func() { sendLogMessages("message 1", inMessageChan) var readMessage *events.Envelope Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage)) Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1")) sendLogMessages("message 2", inMessageChan) Eventually(buffer.GetOutputChannel).Should(Receive(&readMessage)) Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 2")) }) Context("tracking dropped messages", func() { var fakeEventEmitter *fake.FakeEventEmitter BeforeEach(func() { fakeEventEmitter = fake.NewFakeEventEmitter("doppler") sender := metric_sender.NewMetricSender(fakeEventEmitter) batcher := metricbatcher.New(sender, 100*time.Millisecond)
lm, _ = emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "my message", appId, "App"), "origin") }) AfterEach(func() { close(stopKeepAlive) <-connectionDropped cleanup() }) It("sends data to the websocket firehose client", func() { sinkManager.SendTo(appId, lm) rlm, err := receiveEnvelope(wsReceivedChan) Expect(err).NotTo(HaveOccurred()) Expect(rlm.GetLogMessage().GetMessage()).To(Equal(lm.GetLogMessage().GetMessage())) }) Context("when data is sent to the websocket firehose client", func() { JustBeforeEach(func() { sinkManager.SendTo(appId, lm) }) It("emits counter metric sentMessagesFirehose", func() { Eventually(mockBatcher.BatchCounterInput).Should(BeCalled( With("sentMessagesFirehose"), )) Eventually(mockChainer.SetTagInput).Should(BeCalled( With("subscription_id", subscriptionID), ))
bytes := generator.Generate() var envelope events.Envelope err := proto.Unmarshal(bytes, &envelope) Expect(err).ToNot(HaveOccurred()) Expect(envelope.GetEventType()).To(Equal(events.Envelope_ValueMetric)) }) }) Context("The LogGenerator", func() { It("generates a log message envelope", func() { generator := messagegenerator.NewLogMessageGenerator() bytes := generator.Generate() var envelope events.Envelope err := proto.Unmarshal(bytes, &envelope) Expect(err).ToNot(HaveOccurred()) Expect(envelope.GetLogMessage()).ToNot(BeNil()) Expect(envelope.GetLogMessage().GetMessage()).To(BeEquivalentTo("test message")) }) }) Context("The LegacyLogGenerator", func() { It("generates a legacy log message envelope", func() { generator := messagegenerator.NewLegacyLogGenerator() bytes := generator.Generate() var envelope logmessage.LogEnvelope err := proto.Unmarshal(bytes, &envelope) Expect(err).ToNot(HaveOccurred()) Expect(envelope.GetLogMessage()).ToNot(BeNil()) Expect(envelope.GetLogMessage().GetMessage()).To(BeEquivalentTo("test message")) }) })