func dopplerReceivedMessageCount(msg *events.Envelope) bool {
return msg.GetEventType() == events.Envelope_CounterEvent &&
(msg.CounterEvent.GetName() == "udpListener.receivedMessageCount" ||
msg.CounterEvent.GetName() == "tcpListener.receivedMessageCount" ||
msg.CounterEvent.GetName() == "tlsListener.receivedMessageCount") &&
msg.GetOrigin() == "DopplerServer"
}
func HttpStartStop(msg *events.Envelope) Event {
httpStartStop := msg.GetHttpStartStop()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"cf_app_id": httpStartStop.GetApplicationId(),
"content_length": httpStartStop.GetContentLength(),
"instance_id": httpStartStop.GetInstanceId(),
"instance_index": httpStartStop.GetInstanceIndex(),
"method": httpStartStop.GetMethod(),
"parent_request_id": httpStartStop.GetParentRequestId(),
"peer_type": httpStartStop.GetPeerType(),
"remote_addr": httpStartStop.GetRemoteAddress(),
"request_id": httpStartStop.GetRequestId(),
"start_timestamp": httpStartStop.GetStartTimestamp(),
"status_code": httpStartStop.GetStatusCode(),
"stop_timestamp": httpStartStop.GetStopTimestamp(),
"uri": httpStartStop.GetUri(),
"user_agent": httpStartStop.GetUserAgent(),
"duration_ms": (((httpStartStop.GetStopTimestamp() - httpStartStop.GetStartTimestamp()) / 1000) / 1000),
}
return Event{
Fields: fields,
Msg: "",
Type: msg.GetEventType().String(),
}
}
func (e *Event) AnnotateWithEnveloppeData(msg *events.Envelope) {
e.Fields["origin"] = msg.GetOrigin()
e.Fields["deployment"] = msg.GetDeployment()
e.Fields["ip"] = msg.GetIp()
e.Fields["job"] = msg.GetJob()
e.Fields["index"] = msg.GetIndex()
//e.Fields["tags"] = msg.GetTags()
e.Type = msg.GetEventType().String()
}
func (m *MetricsNozzle) handleMessage(envelope *events.Envelope) {
if m.config.OriginID != "" {
if envelope.GetOrigin() == m.config.OriginID {
fmt.Fprintf(m.stdout, "%v \n", envelope)
}
} else {
fmt.Fprintf(m.stdout, "%v \n", envelope)
}
}
func checkValueMetric(receivedEnvelope *events.Envelope, origin string, name string, value float64, unit string) {
Expect(receivedEnvelope.GetEventType()).To(Equal(events.Envelope_ValueMetric))
Expect(receivedEnvelope.GetOrigin()).To(Equal(origin))
vm := receivedEnvelope.GetValueMetric()
Expect(vm.GetName()).To(Equal(name))
Expect(vm.GetValue()).To(BeNumerically("==", value))
Expect(vm.GetUnit()).To(Equal(unit))
}
func getName(envelope *events.Envelope) string {
switch envelope.GetEventType() {
case events.Envelope_ValueMetric:
return envelope.GetOrigin() + "." + envelope.GetValueMetric().GetName()
case events.Envelope_CounterEvent:
return envelope.GetOrigin() + "." + envelope.GetCounterEvent().GetName()
default:
panic("Unknown event type")
}
}
func (m *MsgLossChart) ForChart(event *events.Envelope) bool {
if event.GetEventType() != events.Envelope_CounterEvent {
return false
}
if !contains(event.GetOrigin(), m.validOrigins) {
return false
}
if !contains(event.GetCounterEvent().GetName(), m.validMetricNames) {
return false
}
return true
}
func (s *SinkTypeChart) ForChart(event *events.Envelope) bool {
if event.GetEventType() != events.Envelope_ValueMetric {
return false
}
if !contains(event.GetOrigin(), s.validOrigins) {
return false
}
if !contains(event.GetValueMetric().GetName(), s.validMetricNames) {
return false
}
// s.cfUI.Say("%f ", event.GetValueMetric().GetValue())
return true
}
func (m *MessageAggregator) handleCounter(envelope *events.Envelope) *events.Envelope {
metrics.BatchIncrementCounter("MessageAggregator.counterEventReceived")
countID := counterID{
name: envelope.GetCounterEvent().GetName(),
origin: envelope.GetOrigin(),
}
newVal := m.counterTotals[countID] + envelope.GetCounterEvent().GetDelta()
m.counterTotals[countID] = newVal
envelope.GetCounterEvent().Total = &newVal
return envelope
}
func ErrorEvent(msg *events.Envelope) Event {
errorEvent := msg.GetError()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"code": errorEvent.GetCode(),
"delta": errorEvent.GetSource(),
}
return Event{
Fields: fields,
Msg: errorEvent.GetMessage(),
Type: msg.GetEventType().String(),
}
}
func CounterEvent(msg *events.Envelope) Event {
counterEvent := msg.GetCounterEvent()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"name": counterEvent.GetName(),
"delta": counterEvent.GetDelta(),
"total": counterEvent.GetTotal(),
}
return Event{
Fields: fields,
Msg: "",
Type: msg.GetEventType().String(),
}
}
func ValueMetric(msg *events.Envelope) Event {
valMetric := msg.GetValueMetric()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"name": valMetric.GetName(),
"unit": valMetric.GetUnit(),
"value": valMetric.GetValue(),
}
return Event{
Fields: fields,
Msg: "",
Type: msg.GetEventType().String(),
}
}
func listenForEvents(origin []string) {
for {
buffer := make([]byte, 1024)
n, _, err := udpListener.ReadFrom(buffer)
if err != nil {
return
}
if n == 0 {
panic("Received empty packet")
}
envelope := new(events.Envelope)
err = proto.Unmarshal(buffer[0:n], envelope)
if err != nil {
panic(err)
}
var eventId = envelope.GetEventType().String()
tracker := eventTracker{eventType: eventId}
switch envelope.GetEventType() {
case events.Envelope_HttpStart:
tracker.name = envelope.GetHttpStart().GetPeerType().String()
case events.Envelope_HttpStop:
tracker.name = envelope.GetHttpStop().GetPeerType().String()
case events.Envelope_ValueMetric:
tracker.name = envelope.GetValueMetric().GetName()
case events.Envelope_CounterEvent:
tracker.name = envelope.GetCounterEvent().GetName()
default:
panic("Unexpected message type")
}
if envelope.GetOrigin() != strings.Join(origin, "/") {
panic("origin not as expected")
}
func() {
lock.Lock()
defer lock.Unlock()
receivedEvents = append(receivedEvents, tracker)
}()
}
}
func LogMessage(msg *events.Envelope) Event {
logMessage := msg.GetLogMessage()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"cf_app_id": logMessage.GetAppId(),
"timestamp": logMessage.GetTimestamp(),
"source_type": logMessage.GetSourceType(),
"message_type": logMessage.GetMessageType().String(),
"source_instance": logMessage.GetSourceInstance(),
}
return Event{
Fields: fields,
Msg: string(logMessage.GetMessage()),
Type: msg.GetEventType().String(),
}
}
func ContainerMetric(msg *events.Envelope) Event {
containerMetric := msg.GetContainerMetric()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"cf_app_id": containerMetric.GetApplicationId(),
"cpu_percentage": containerMetric.GetCpuPercentage(),
"disk_bytes": containerMetric.GetDiskBytes(),
"instance_index": containerMetric.GetInstanceIndex(),
"memory_bytes": containerMetric.GetMemoryBytes(),
}
return Event{
Fields: fields,
Msg: "",
Type: msg.GetEventType().String(),
}
}
func (vf *VarzForwarder) addMetric(metric *events.Envelope) {
vf.lock.Lock()
defer vf.lock.Unlock()
originMetrics, ok := vf.metricsByOrigin[metric.GetOrigin()]
if !ok {
vf.metricsByOrigin[metric.GetOrigin()] = vf.createMetrics(metric.GetOrigin())
originMetrics = vf.metricsByOrigin[metric.GetOrigin()]
}
originMetrics.processMetric(metric)
}
func HttpStop(msg *events.Envelope) Event {
httpStop := msg.GetHttpStop()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"cf_app_id": httpStop.GetApplicationId(),
"content_length": httpStop.GetContentLength(),
"peer_type": httpStop.GetPeerType(),
"request_id": httpStop.GetRequestId(),
"status_code": httpStop.GetStatusCode(),
"timestamp": httpStop.GetTimestamp(),
"uri": httpStop.GetUri(),
}
return Event{
Fields: fields,
Msg: "",
Type: msg.GetEventType().String(),
}
}
//EventToJSON turns a firehose event into a json representation
func EventToJSON(event *events.Envelope) *[]byte {
props := map[string]interface{}{
"time": event.GetTimestamp() / 1000000000,
"origin": event.GetOrigin(),
"deployment": event.GetDeployment(),
"job": event.GetJob(),
"index": event.GetIndex(),
"ip": event.GetIp(),
"token": mixPanelToken,
}
data := map[string]interface{}{
"event": event.GetEventType().String(),
"properties": props,
}
j, err := json.Marshal(data)
if nil != err {
log.Print("Failed to marshal event")
log.Print(data)
}
return &j
}
func HttpStart(msg *events.Envelope) Event {
httpStart := msg.GetHttpStart()
fields := logrus.Fields{
"origin": msg.GetOrigin(),
"cf_app_id": httpStart.GetApplicationId(),
"instance_id": httpStart.GetInstanceId(),
"instance_index": httpStart.GetInstanceIndex(),
"method": httpStart.GetMethod(),
"parent_request_id": httpStart.GetParentRequestId(),
"peer_type": httpStart.GetPeerType(),
"request_id": httpStart.GetRequestId(),
"remote_addr": httpStart.GetRemoteAddress(),
"timestamp": httpStart.GetTimestamp(),
"uri": httpStart.GetUri(),
"user_agent": httpStart.GetUserAgent(),
}
return Event{
Fields: fields,
Msg: "",
Type: msg.GetEventType().String(),
}
}
func (vf *VarzForwarder) Write(envelope *events.Envelope) {
vf.addMetric(envelope)
vf.resetTimer(envelope.GetOrigin())
vf.outputWriter.Write(envelope)
}
func metronSentMessageCount(msg *events.Envelope) bool {
return msg.GetEventType() == events.Envelope_CounterEvent && msg.CounterEvent.GetName() == "DopplerForwarder.sentMessages" && msg.GetOrigin() == "MetronAgent"
}
func dopplerSentMessageCount(msg *events.Envelope) bool {
return msg.GetEventType() == events.Envelope_CounterEvent && strings.HasPrefix(msg.CounterEvent.GetName(), "sentMessagesFirehose") && msg.GetOrigin() == "DopplerServer"
}
func isMetronMessageCount(msg *events.Envelope) bool {
return msg.GetEventType() == events.Envelope_CounterEvent && msg.CounterEvent.GetName() == "dropsondeAgentListener.receivedMessageCount" && msg.GetOrigin() == "MetronAgent"
}
func matchCounter(expected, actual *events.Envelope) bool {
return expected.GetOrigin() == actual.GetOrigin() &&
expected.GetEventType() == actual.GetEventType() &&
expected.GetCounterEvent().GetName() == actual.GetCounterEvent().GetName() &&
actual.GetCounterEvent().GetTotal() > 0
}
Context("single StartStop message", func() {
var outputMessage *events.Envelope
BeforeEach(func() {
messageAggregator.Write(createStartMessage(123, events.PeerType_Client))
messageAggregator.Write(createStopMessage(123, events.PeerType_Client))
outputMessage = mockWriter.Events[0]
})
It("populates all fields in the StartStop message correctly", func() {
Expect(outputMessage.GetHttpStartStop()).To(Equal(createStartStopMessage(123, events.PeerType_Client).GetHttpStartStop()))
})
It("populates all fields in the Envelope correctly", func() {
Expect(outputMessage.GetOrigin()).To(Equal("fake-origin-2"))
Expect(outputMessage.GetTimestamp()).ToNot(BeZero())
Expect(outputMessage.GetEventType()).To(Equal(events.Envelope_HttpStartStop))
})
})
It("does not send a combined event if there only is a stop event", func() {
messageAggregator.Write(createStopMessage(123, events.PeerType_Client))
Consistently(func() int { return len(mockWriter.Events) }).Should(Equal(0))
})
Context("message expiry", func() {
BeforeEach(func() {
messageaggregator.MaxTTL = 0
})
func isDopplerMessageCount(msg *events.Envelope) bool {
return msg.GetEventType() == events.Envelope_CounterEvent && msg.CounterEvent.GetName() == "dropsondeListener.receivedMessageCount" && msg.GetOrigin() == "DopplerServer"
}
case events.Envelope_HttpStart:
tracker.name = envelope.GetHttpStart().GetPeerType().String()
case events.Envelope_HttpStop:
tracker.name = envelope.GetHttpStop().GetPeerType().String()
case events.Envelope_Heartbeat:
tracker.name = envelope.GetHeartbeat().GetControlMessageIdentifier().String()
case events.Envelope_ValueMetric:
tracker.name = envelope.GetValueMetric().GetName()
case events.Envelope_CounterEvent:
tracker.name = envelope.GetCounterEvent().GetName()
default:
panic("Unexpected message type")
}
if envelope.GetOrigin() != strings.Join(origin, "/") {
panic("origin not as expected")
}
func() {
lock.Lock()
defer lock.Unlock()
receivedEvents = append(receivedEvents, tracker)
}()
}
}()
httpListener, err := net.Listen("tcp", "localhost:0")
Expect(err).ToNot(HaveOccurred())
defer httpListener.Close()
httpHandler := dropsonde.InstrumentedHandler(FakeHandler{})
func (d *OpenTSDBFirehoseNozzle) handleMessage(envelope *events.Envelope) {
if envelope.GetEventType() == events.Envelope_CounterEvent && envelope.CounterEvent.GetName() == "TruncatingBuffer.DroppedMessages" && envelope.GetOrigin() == "doppler" {
log.Printf("We've intercepted an upstream message which indicates that the nozzle or the TrafficController is not keeping up. Please try scaling up the nozzle.")
d.client.AlertSlowConsumerError()
}
}
func TranslateDropsondeToLegacyLogMessage(message []byte) ([]byte, error) {
var receivedEnvelope events.Envelope
err := proto.Unmarshal(message, &receivedEnvelope)
if err != nil {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Unable to unmarshal bytes as Envelope: %v", err)
}
if receivedEnvelope.GetEventType() != events.Envelope_LogMessage {
return nil, nil
}
logMessage := receivedEnvelope.GetLogMessage()
if logMessage == nil {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope's LogMessage was nil with eventType: %v and origin: %v", receivedEnvelope.GetEventType(), receivedEnvelope.GetOrigin())
}
messageBytes, err := proto.Marshal(
&logmessage.LogMessage{
Message: logMessage.GetMessage(),
MessageType: (*logmessage.LogMessage_MessageType)(logMessage.MessageType),
Timestamp: proto.Int64(logMessage.GetTimestamp()),
AppId: proto.String(logMessage.GetAppId()),
SourceId: proto.String(logMessage.GetSourceInstance()),
SourceName: proto.String(logMessage.GetSourceType()),
},
)
if err != nil {
return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Failed marshalling converted dropsonde message: %v", err)
}
return messageBytes, nil
}