func (m *MessageAggregator) Write(envelope *events.Envelope) { // TODO: don't call for every message if throughput becomes a problem m.cleanupOrphanedHTTPStart() if envelope.EventType == nil { m.outputWriter.Write(envelope) return } switch envelope.GetEventType() { case events.Envelope_HttpStart: m.handleHTTPStart(envelope) case events.Envelope_HttpStop: startStopMessage := m.handleHTTPStop(envelope) if startStopMessage != nil { m.outputWriter.Write(startStopMessage) } case events.Envelope_CounterEvent: counterEventMessage := m.handleCounter(envelope) m.outputWriter.Write(counterEventMessage) default: m.incrementCounter(&m.uncategorizedEventCount) m.logger.Debugf("passing through message %v", spew.Sprintf("%v", envelope)) m.outputWriter.Write(envelope) } }
func (m *MessageAggregator) Write(envelope *events.Envelope) { // TODO: don't call for every message if throughput becomes a problem m.cleanupOrphanedHTTPStart() if envelope.EventType == nil { metrics.BatchIncrementCounter("MessageAggregator.uncategorizedEvents") return } switch envelope.GetEventType() { case events.Envelope_HttpStart: m.handleHTTPStart(envelope) case events.Envelope_HttpStop: startStopMessage := m.handleHTTPStop(envelope) if startStopMessage != nil { m.outputWriter.Write(startStopMessage) } case events.Envelope_CounterEvent: counterEventMessage := m.handleCounter(envelope) m.outputWriter.Write(counterEventMessage) default: metrics.BatchIncrementCounter("MessageAggregator.uncategorizedEvents") m.outputWriter.Write(envelope) } }
func HttpStartStop(msg *events.Envelope) Event { httpStartStop := msg.GetHttpStartStop() fields := logrus.Fields{ "origin": msg.GetOrigin(), "cf_app_id": httpStartStop.GetApplicationId(), "content_length": httpStartStop.GetContentLength(), "instance_id": httpStartStop.GetInstanceId(), "instance_index": httpStartStop.GetInstanceIndex(), "method": httpStartStop.GetMethod(), "parent_request_id": httpStartStop.GetParentRequestId(), "peer_type": httpStartStop.GetPeerType(), "remote_addr": httpStartStop.GetRemoteAddress(), "request_id": httpStartStop.GetRequestId(), "start_timestamp": httpStartStop.GetStartTimestamp(), "status_code": httpStartStop.GetStatusCode(), "stop_timestamp": httpStartStop.GetStopTimestamp(), "uri": httpStartStop.GetUri(), "user_agent": httpStartStop.GetUserAgent(), "duration_ms": (((httpStartStop.GetStopTimestamp() - httpStartStop.GetStartTimestamp()) / 1000) / 1000), } return Event{ Fields: fields, Msg: "", Type: msg.GetEventType().String(), } }
func dopplerReceivedMessageCount(msg *events.Envelope) bool { return msg.GetEventType() == events.Envelope_CounterEvent && (msg.CounterEvent.GetName() == "udpListener.receivedMessageCount" || msg.CounterEvent.GetName() == "tcpListener.receivedMessageCount" || msg.CounterEvent.GetName() == "tlsListener.receivedMessageCount") && msg.GetOrigin() == "DopplerServer" }
func (l *logCounter) processEnvelope(env *events.Envelope) { if env.GetEventType() != events.Envelope_LogMessage { return } logMsg := env.GetLogMessage() msg := string(logMsg.GetMessage()) if strings.HasPrefix(msg, "mismatched prefix") { return } if !strings.HasPrefix(msg, l.cfg.MessagePrefix) { fmt.Printf("mismatched prefix: log message %s did not match prefix: %s\n", string(logMsg.GetMessage()), string(l.cfg.MessagePrefix)) return } if len(msg) < sepEnd { fmt.Printf("Cannot parse message %s\n", msg) return } id := Identity{ appID: logMsg.GetAppId(), runID: msg[prefixEnd:guidEnd], } l.counterLock.Lock() defer l.counterLock.Unlock() counter, ok := l.counters[id] if !ok { counter = make(map[string]bool) l.counters[id] = counter } counter[msg[sepEnd:]] = true }
func TranslateDropsondeToLegacyLogMessage(message []byte) ([]byte, error) { var receivedEnvelope events.Envelope err := proto.Unmarshal(message, &receivedEnvelope) if err != nil { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Unable to unmarshal bytes as Envelope: %v", err) } if receivedEnvelope.GetEventType() != events.Envelope_LogMessage { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope contained %s instead of LogMessage", receivedEnvelope.GetEventType().String()) } logMessage := receivedEnvelope.GetLogMessage() if logMessage == nil { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope's LogMessage was nil: %v", receivedEnvelope) } messageBytes, err := proto.Marshal( &logmessage.LogMessage{ Message: logMessage.GetMessage(), MessageType: (*logmessage.LogMessage_MessageType)(logMessage.MessageType), Timestamp: proto.Int64(logMessage.GetTimestamp()), AppId: proto.String(logMessage.GetAppId()), SourceId: proto.String(logMessage.GetSourceInstance()), SourceName: proto.String(logMessage.GetSourceType()), }, ) if err != nil { return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Failed marshalling converted dropsonde message: %v", err) } return messageBytes, nil }
func routeEvent(msg *events.Envelope, extraFields map[string]string) { eventType := msg.GetEventType() if selectedEvents[eventType.String()] { var event Event switch eventType { case events.Envelope_HttpStart: event = HttpStart(msg) case events.Envelope_HttpStop: event = HttpStop(msg) case events.Envelope_HttpStartStop: event = HttpStartStop(msg) case events.Envelope_LogMessage: event = LogMessage(msg) case events.Envelope_ValueMetric: event = ValueMetric(msg) case events.Envelope_CounterEvent: event = CounterEvent(msg) case events.Envelope_Error: event = ErrorEvent(msg) case events.Envelope_ContainerMetric: event = ContainerMetric(msg) } event.AnnotateWithAppData() event.AnnotateWithMetaData(extraFields) event.ShipEvent() mutex.Lock() selectedEventsCount[eventType.String()]++ mutex.Unlock() } }
func listenForLogs() { for { buffer := make([]byte, 1024) n, _, err := udpConn.ReadFrom(buffer) if err != nil { return } if n == 0 { panic("Received empty packet") } envelope := new(events.Envelope) err = proto.Unmarshal(buffer[0:n], envelope) if err != nil { panic(err) } if envelope.GetEventType() == events.Envelope_LogMessage { logLock.Lock() logMessages = append(logMessages, envelope.GetLogMessage()) logLock.Unlock() } } }
func (d *DopplerForwarder) networkWrite(message *events.Envelope) { client, err := d.clientPool.RandomClient() if err != nil { d.logger.Errord(map[string]interface{}{ "error": err.Error(), }, "DopplerForwarder: can't forward message") return } messageBytes, err := proto.Marshal(message) if err != nil { d.logger.Errorf("DopplerForwarder: marshal error %v", err) metrics.BatchIncrementCounter("dropsondeMarshaller.marshalErrors") return } switch client.Scheme() { case "udp": signedMessage := signature.SignMessage(messageBytes, d.sharedSecret) bytesWritten, err := client.Write(signedMessage) if err != nil { metrics.BatchIncrementCounter("udp.sendErrorCount") d.logger.Debugd(map[string]interface{}{ "scheme": client.Scheme(), "address": client.Address(), }, "Error writing legacy message") return } metrics.BatchIncrementCounter("udp.sentMessageCount") metrics.BatchAddCounter("udp.sentByteCount", uint64(bytesWritten)) case "tls": var bytesWritten int err = binary.Write(client, binary.LittleEndian, uint32(len(messageBytes))) if err == nil { bytesWritten, err = client.Write(messageBytes) } if err != nil { metrics.BatchIncrementCounter("tls.retryCount") d.inputChan <- message client.Close() d.logger.Errord(map[string]interface{}{ "scheme": client.Scheme(), "address": client.Address(), "error": err.Error(), }, "DopplerForwarder: streaming error") return } metrics.BatchIncrementCounter("tls.sentMessageCount") metrics.BatchAddCounter("tls.sentByteCount", uint64(bytesWritten+4)) default: d.logger.Errorf("DopplerForwarder: unknown protocol, %s for %s", client.Scheme(), client.Address()) return } d.incrementMessageCount(message.GetEventType()) metrics.BatchIncrementCounter("DopplerForwarder.sentMessages") }
func getValue(envelope *events.Envelope) float64 { switch envelope.GetEventType() { case events.Envelope_ValueMetric: return envelope.GetValueMetric().GetValue() case events.Envelope_CounterEvent: return float64(envelope.GetCounterEvent().GetTotal()) default: panic("Unknown event type") } }
func getName(envelope *events.Envelope) string { switch envelope.GetEventType() { case events.Envelope_ValueMetric: return envelope.GetOrigin() + "." + envelope.GetValueMetric().GetName() case events.Envelope_CounterEvent: return envelope.GetOrigin() + "." + envelope.GetCounterEvent().GetName() default: panic("Unknown event type") } }
func (metrics *metrics) processMetric(metric *events.Envelope) { switch metric.GetEventType() { case events.Envelope_ValueMetric: metrics.processValueMetric(metric) case events.Envelope_CounterEvent: metrics.processCounterEvent(metric) case events.Envelope_HttpStartStop: metrics.processHTTPStartStop(metric) } }
func (e *Event) AnnotateWithEnveloppeData(msg *events.Envelope) { e.Fields["origin"] = msg.GetOrigin() e.Fields["deployment"] = msg.GetDeployment() e.Fields["ip"] = msg.GetIp() e.Fields["job"] = msg.GetJob() e.Fields["index"] = msg.GetIndex() //e.Fields["tags"] = msg.GetTags() e.Type = msg.GetEventType().String() }
func checkValueMetric(receivedEnvelope *events.Envelope, origin string, name string, value float64, unit string) { Expect(receivedEnvelope.GetEventType()).To(Equal(events.Envelope_ValueMetric)) Expect(receivedEnvelope.GetOrigin()).To(Equal(origin)) vm := receivedEnvelope.GetValueMetric() Expect(vm.GetName()).To(Equal(name)) Expect(vm.GetValue()).To(BeNumerically("==", value)) Expect(vm.GetUnit()).To(Equal(unit)) }
func (f *fakeWriter) Write(message *events.Envelope) { if message.GetEventType() == events.Envelope_ValueMetric { switch message.GetValueMetric().GetName() { case "Uptime": atomic.StoreUint64(&f.lastUptime, uint64(message.GetValueMetric().GetValue())) case "LinuxFileDescriptor": atomic.StoreUint64(&f.openFileDescriptors, uint64(message.GetValueMetric().GetValue())) } } }
func (m *MsgLossChart) ForChart(event *events.Envelope) bool { if event.GetEventType() != events.Envelope_CounterEvent { return false } if !contains(event.GetOrigin(), m.validOrigins) { return false } if !contains(event.GetCounterEvent().GetName(), m.validMetricNames) { return false } return true }
func (m *EventMarshaller) Write(message *events.Envelope) { messageBytes, err := proto.Marshal(message) if err != nil { m.logger.Errorf("eventMarshaller: marshal error %v for message %v", err, message) metrics.BatchIncrementCounter("dropsondeMarshaller.marshalErrors") return } m.logger.Debugf("eventMarshaller: marshalled message %v", spew.Sprintf("%v", message)) m.incrementMessageCount(message.GetEventType()) m.outputWriter.Write(messageBytes) }
func (s *SinkTypeChart) ForChart(event *events.Envelope) bool { if event.GetEventType() != events.Envelope_ValueMetric { return false } if !contains(event.GetOrigin(), s.validOrigins) { return false } if !contains(event.GetValueMetric().GetName(), s.validMetricNames) { return false } // s.cfUI.Say("%f ", event.GetValueMetric().GetValue()) return true }
func (c *Client) AddMetric(envelope *events.Envelope) { c.totalMessagesReceived++ if envelope.GetEventType() != events.Envelope_ValueMetric && envelope.GetEventType() != events.Envelope_CounterEvent { return } metric := poster.Metric{ Value: getValue(envelope), Timestamp: envelope.GetTimestamp() / int64(time.Second), Metric: c.prefix + getName(envelope), Tags: getTags(envelope), } c.metrics = append(c.metrics, metric) }
func ErrorEvent(msg *events.Envelope) Event { errorEvent := msg.GetError() fields := logrus.Fields{ "origin": msg.GetOrigin(), "code": errorEvent.GetCode(), "delta": errorEvent.GetSource(), } return Event{ Fields: fields, Msg: errorEvent.GetMessage(), Type: msg.GetEventType().String(), } }
func updateAndReturnValue(values map[string]int, event *events.Envelope) int { switch event.GetEventType() { case events.Envelope_CounterEvent: values[event.GetIp()] = int(event.GetCounterEvent().GetTotal()) case events.Envelope_ValueMetric: values[event.GetIp()] = int(event.GetValueMetric().GetValue()) } sum := 0 for _, v := range values { sum += v } return sum }
func ValueMetric(msg *events.Envelope) Event { valMetric := msg.GetValueMetric() fields := logrus.Fields{ "origin": msg.GetOrigin(), "name": valMetric.GetName(), "unit": valMetric.GetUnit(), "value": valMetric.GetValue(), } return Event{ Fields: fields, Msg: "", Type: msg.GetEventType().String(), } }
func CounterEvent(msg *events.Envelope) Event { counterEvent := msg.GetCounterEvent() fields := logrus.Fields{ "origin": msg.GetOrigin(), "name": counterEvent.GetName(), "delta": counterEvent.GetDelta(), "total": counterEvent.GetTotal(), } return Event{ Fields: fields, Msg: "", Type: msg.GetEventType().String(), } }
func routeEvent(msg *events.Envelope, extraFields map[string]string, filters filters.LogFilters) { eventType := msg.GetEventType() if selectedEvents[eventType.String()] { var event Event switch eventType { case events.Envelope_HttpStart: event = HttpStart(msg) case events.Envelope_HttpStop: event = HttpStop(msg) case events.Envelope_HttpStartStop: event = HttpStartStop(msg) case events.Envelope_LogMessage: event = LogMessage(msg) case events.Envelope_ValueMetric: event = ValueMetric(msg) case events.Envelope_CounterEvent: event = CounterEvent(msg) case events.Envelope_Error: event = ErrorEvent(msg) case events.Envelope_ContainerMetric: event = ContainerMetric(msg) } event.AnnotateWithAppData() event.AnnotateWithMetaData(extraFields) if event.ApplyFilters(filters) { // Compose the key to be used in Redis orgId := event.Fields["cf_org_id"].(string) orgEnabledKey := orgId + ":enabled" // Get the "enabled" flag to know if this organization will keep sending logs to ELK orgEnabled, _ := redis.Bool(redisConn.Do("HGET", redisHash, orgEnabledKey)) if orgEnabled { // Increment the value of the organization key to add the amount of bytes of the log value, err := redisConn.Do("HINCRBY", redisHash, orgId, len(event.Msg)) if err != nil || value == nil { panic("Redis server could not be contacted!") } // Send the event (log message) event.ShipEvent() } } } }
func valid(env *events.Envelope) bool { switch env.GetEventType() { case events.Envelope_HttpStartStop: return env.GetHttpStartStop() != nil case events.Envelope_LogMessage: return env.GetLogMessage() != nil case events.Envelope_ValueMetric: return env.GetValueMetric() != nil case events.Envelope_CounterEvent: return env.GetCounterEvent() != nil case events.Envelope_Error: return env.GetError() != nil case events.Envelope_ContainerMetric: return env.GetContainerMetric() != nil } return true }
func listenForEvents(origin []string) { for { buffer := make([]byte, 1024) n, _, err := udpListener.ReadFrom(buffer) if err != nil { return } if n == 0 { panic("Received empty packet") } envelope := new(events.Envelope) err = proto.Unmarshal(buffer[0:n], envelope) if err != nil { panic(err) } var eventId = envelope.GetEventType().String() tracker := eventTracker{eventType: eventId} switch envelope.GetEventType() { case events.Envelope_HttpStart: tracker.name = envelope.GetHttpStart().GetPeerType().String() case events.Envelope_HttpStop: tracker.name = envelope.GetHttpStop().GetPeerType().String() case events.Envelope_ValueMetric: tracker.name = envelope.GetValueMetric().GetName() case events.Envelope_CounterEvent: tracker.name = envelope.GetCounterEvent().GetName() default: panic("Unexpected message type") } if envelope.GetOrigin() != strings.Join(origin, "/") { panic("origin not as expected") } func() { lock.Lock() defer lock.Unlock() receivedEvents = append(receivedEvents, tracker) }() } }
func LogMessage(msg *events.Envelope) Event { logMessage := msg.GetLogMessage() fields := logrus.Fields{ "origin": msg.GetOrigin(), "cf_app_id": logMessage.GetAppId(), "timestamp": logMessage.GetTimestamp(), "source_type": logMessage.GetSourceType(), "message_type": logMessage.GetMessageType().String(), "source_instance": logMessage.GetSourceInstance(), } return Event{ Fields: fields, Msg: string(logMessage.GetMessage()), Type: msg.GetEventType().String(), } }
func ContainerMetric(msg *events.Envelope) Event { containerMetric := msg.GetContainerMetric() fields := logrus.Fields{ "origin": msg.GetOrigin(), "cf_app_id": containerMetric.GetApplicationId(), "cpu_percentage": containerMetric.GetCpuPercentage(), "disk_bytes": containerMetric.GetDiskBytes(), "instance_index": containerMetric.GetInstanceIndex(), "memory_bytes": containerMetric.GetMemoryBytes(), } return Event{ Fields: fields, Msg: "", Type: msg.GetEventType().String(), } }
func (m *EventMarshaller) Write(envelope *events.Envelope) { writer := m.writer() if writer == nil { m.logger.Warn("EventMarshaller: Write called while byteWriter is nil") return } envelopeBytes, err := proto.Marshal(envelope) if err != nil { m.logger.Errorf("marshalling error: %v", err) m.batcher.BatchIncrementCounter("dropsondeMarshaller.marshalErrors") return } chainer := m.batcher.BatchCounter("dropsondeMarshaller.sentEnvelopes"). SetTag("event_type", envelope.GetEventType().String()) writer.Write(envelopeBytes, chainer) }
func HttpStop(msg *events.Envelope) Event { httpStop := msg.GetHttpStop() fields := logrus.Fields{ "origin": msg.GetOrigin(), "cf_app_id": httpStop.GetApplicationId(), "content_length": httpStop.GetContentLength(), "peer_type": httpStop.GetPeerType(), "request_id": httpStop.GetRequestId(), "status_code": httpStop.GetStatusCode(), "timestamp": httpStop.GetTimestamp(), "uri": httpStop.GetUri(), } return Event{ Fields: fields, Msg: "", Type: msg.GetEventType().String(), } }