Example #1
0
func routeEvent(msg *events.Envelope, extraFields map[string]string) {

	eventType := msg.GetEventType()

	if selectedEvents[eventType.String()] {
		var event Event
		switch eventType {
		case events.Envelope_HttpStart:
			event = HttpStart(msg)
		case events.Envelope_HttpStop:
			event = HttpStop(msg)
		case events.Envelope_HttpStartStop:
			event = HttpStartStop(msg)
		case events.Envelope_LogMessage:
			event = LogMessage(msg)
		case events.Envelope_ValueMetric:
			event = ValueMetric(msg)
		case events.Envelope_CounterEvent:
			event = CounterEvent(msg)
		case events.Envelope_Error:
			event = ErrorEvent(msg)
		case events.Envelope_ContainerMetric:
			event = ContainerMetric(msg)
		}

		event.AnnotateWithAppData()
		event.AnnotateWithMetaData(extraFields)
		event.ShipEvent()

		mutex.Lock()
		selectedEventsCount[eventType.String()]++
		mutex.Unlock()
	}
}
Example #2
0
func HttpStartStop(msg *events.Envelope) Event {
	httpStartStop := msg.GetHttpStartStop()

	fields := logrus.Fields{
		"cf_app_id":       utils.FormatUUID(httpStartStop.GetApplicationId()),
		"content_length":  httpStartStop.GetContentLength(),
		"instance_id":     httpStartStop.GetInstanceId(),
		"instance_index":  httpStartStop.GetInstanceIndex(),
		"method":          httpStartStop.GetMethod(),
		"peer_type":       httpStartStop.GetPeerType(),
		"remote_addr":     httpStartStop.GetRemoteAddress(),
		"request_id":      utils.FormatUUID(httpStartStop.GetRequestId()),
		"start_timestamp": httpStartStop.GetStartTimestamp(),
		"status_code":     httpStartStop.GetStatusCode(),
		"stop_timestamp":  httpStartStop.GetStopTimestamp(),
		"uri":             httpStartStop.GetUri(),
		"user_agent":      httpStartStop.GetUserAgent(),
		"duration_ms":     (((httpStartStop.GetStopTimestamp() - httpStartStop.GetStartTimestamp()) / 1000) / 1000),
	}

	return Event{
		Fields: fields,
		Msg:    "",
	}
}
func dopplerReceivedMessageCount(msg *events.Envelope) bool {
	return msg.GetEventType() == events.Envelope_CounterEvent &&
		(msg.CounterEvent.GetName() == "udpListener.receivedMessageCount" ||
			msg.CounterEvent.GetName() == "tcpListener.receivedMessageCount" ||
			msg.CounterEvent.GetName() == "tlsListener.receivedMessageCount") &&
		msg.GetOrigin() == "DopplerServer"
}
func listenForLogs() {
	for {
		buffer := make([]byte, 1024)
		n, _, err := udpConn.ReadFrom(buffer)
		if err != nil {
			return
		}

		if n == 0 {
			panic("Received empty packet")
		}
		envelope := new(events.Envelope)
		err = proto.Unmarshal(buffer[0:n], envelope)
		if err != nil {
			panic(err)
		}

		if envelope.GetEventType() == events.Envelope_LogMessage {
			logLock.Lock()
			logMessages = append(logMessages, envelope.GetLogMessage())
			logLock.Unlock()

		}
	}
}
Example #5
0
func (l *logCounter) processEnvelope(env *events.Envelope) {
	if env.GetEventType() != events.Envelope_LogMessage {
		return
	}
	logMsg := env.GetLogMessage()

	msg := string(logMsg.GetMessage())
	if strings.HasPrefix(msg, "mismatched prefix") {
		return
	}
	if !strings.HasPrefix(msg, l.cfg.MessagePrefix) {
		fmt.Printf("mismatched prefix: log message %s did not match prefix: %s\n", string(logMsg.GetMessage()), string(l.cfg.MessagePrefix))
		return
	}

	if len(msg) < sepEnd {
		fmt.Printf("Cannot parse message %s\n", msg)
		return
	}

	id := Identity{
		appID: logMsg.GetAppId(),
		runID: msg[prefixEnd:guidEnd],
	}

	l.counterLock.Lock()
	defer l.counterLock.Unlock()
	counter, ok := l.counters[id]
	if !ok {
		counter = make(map[string]bool)
		l.counters[id] = counter
	}
	counter[msg[sepEnd:]] = true
}
func TranslateDropsondeToLegacyLogMessage(message []byte) ([]byte, error) {
	var receivedEnvelope events.Envelope
	err := proto.Unmarshal(message, &receivedEnvelope)
	if err != nil {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Unable to unmarshal bytes as Envelope: %v", err)
	}

	if receivedEnvelope.GetEventType() != events.Envelope_LogMessage {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope contained %s instead of LogMessage", receivedEnvelope.GetEventType().String())
	}

	logMessage := receivedEnvelope.GetLogMessage()
	if logMessage == nil {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope's LogMessage was nil: %v", receivedEnvelope)
	}

	messageBytes, err := proto.Marshal(
		&logmessage.LogMessage{
			Message:     logMessage.GetMessage(),
			MessageType: (*logmessage.LogMessage_MessageType)(logMessage.MessageType),
			Timestamp:   proto.Int64(logMessage.GetTimestamp()),
			AppId:       proto.String(logMessage.GetAppId()),
			SourceId:    proto.String(logMessage.GetSourceInstance()),
			SourceName:  proto.String(logMessage.GetSourceType()),
		},
	)
	if err != nil {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Failed marshalling converted dropsonde message: %v", err)
	}

	return messageBytes, nil
}
func (m *MessageAggregator) Write(envelope *events.Envelope) {
	// TODO: don't call for every message if throughput becomes a problem
	m.cleanupOrphanedHTTPStart()

	if envelope.EventType == nil {
		metrics.BatchIncrementCounter("MessageAggregator.uncategorizedEvents")
		return
	}

	switch envelope.GetEventType() {
	case events.Envelope_HttpStart:
		m.handleHTTPStart(envelope)
	case events.Envelope_HttpStop:
		startStopMessage := m.handleHTTPStop(envelope)
		if startStopMessage != nil {
			m.outputWriter.Write(startStopMessage)
		}
	case events.Envelope_CounterEvent:
		counterEventMessage := m.handleCounter(envelope)
		m.outputWriter.Write(counterEventMessage)
	default:
		metrics.BatchIncrementCounter("MessageAggregator.uncategorizedEvents")
		m.outputWriter.Write(envelope)
	}
}
func (m *MessageAggregator) Write(envelope *events.Envelope) {
	// TODO: don't call for every message if throughput becomes a problem
	m.cleanupOrphanedHTTPStart()

	if envelope.EventType == nil {
		m.outputWriter.Write(envelope)
		return
	}
	switch envelope.GetEventType() {
	case events.Envelope_HttpStart:
		m.handleHTTPStart(envelope)
	case events.Envelope_HttpStop:
		startStopMessage := m.handleHTTPStop(envelope)
		if startStopMessage != nil {
			m.outputWriter.Write(startStopMessage)
		}
	case events.Envelope_CounterEvent:
		counterEventMessage := m.handleCounter(envelope)
		m.outputWriter.Write(counterEventMessage)
	default:
		m.incrementCounter(&m.uncategorizedEventCount)
		m.logger.Debugf("passing through message %v", spew.Sprintf("%v", envelope))
		m.outputWriter.Write(envelope)
	}
}
func (d *DopplerForwarder) networkWrite(message *events.Envelope) {
	client, err := d.clientPool.RandomClient()
	if err != nil {
		d.logger.Errord(map[string]interface{}{
			"error": err.Error(),
		}, "DopplerForwarder: can't forward message")
		return
	}

	messageBytes, err := proto.Marshal(message)
	if err != nil {
		d.logger.Errorf("DopplerForwarder: marshal error %v", err)
		metrics.BatchIncrementCounter("dropsondeMarshaller.marshalErrors")
		return
	}

	switch client.Scheme() {
	case "udp":
		signedMessage := signature.SignMessage(messageBytes, d.sharedSecret)
		bytesWritten, err := client.Write(signedMessage)
		if err != nil {
			metrics.BatchIncrementCounter("udp.sendErrorCount")
			d.logger.Debugd(map[string]interface{}{
				"scheme":  client.Scheme(),
				"address": client.Address(),
			}, "Error writing legacy message")
			return
		}
		metrics.BatchIncrementCounter("udp.sentMessageCount")
		metrics.BatchAddCounter("udp.sentByteCount", uint64(bytesWritten))
	case "tls":
		var bytesWritten int
		err = binary.Write(client, binary.LittleEndian, uint32(len(messageBytes)))
		if err == nil {
			bytesWritten, err = client.Write(messageBytes)
		}

		if err != nil {
			metrics.BatchIncrementCounter("tls.retryCount")
			d.inputChan <- message
			client.Close()

			d.logger.Errord(map[string]interface{}{
				"scheme":  client.Scheme(),
				"address": client.Address(),
				"error":   err.Error(),
			}, "DopplerForwarder: streaming error")
			return
		}
		metrics.BatchIncrementCounter("tls.sentMessageCount")
		metrics.BatchAddCounter("tls.sentByteCount", uint64(bytesWritten+4))
	default:
		d.logger.Errorf("DopplerForwarder: unknown protocol, %s for %s", client.Scheme(), client.Address())
		return
	}

	d.incrementMessageCount(message.GetEventType())
	metrics.BatchIncrementCounter("DopplerForwarder.sentMessages")
}
Example #10
0
func addDefaultTags(envelope *events.Envelope) *events.Envelope {
	envelope.Deployment = proto.String("deployment-name")
	envelope.Job = proto.String("test-component")
	envelope.Index = proto.String("42")
	envelope.Ip = proto.String(localIPAddress)

	return envelope
}
Example #11
0
func DecodeProtoBufLogMessage(actual []byte) *events.LogMessage {
	var receivedEnvelope events.Envelope
	err := proto.Unmarshal(actual, &receivedEnvelope)
	if err != nil {
		Fail(err.Error())
	}
	return receivedEnvelope.GetLogMessage()
}
Example #12
0
func EmitToMetron(envelope *events.Envelope) {
	metronConn, err := net.Dial("udp4", fmt.Sprintf("localhost:%d", config.DropsondePort))
	Expect(err).NotTo(HaveOccurred())

	b, err := envelope.Marshal()
	Expect(err).NotTo(HaveOccurred())

	_, err = metronConn.Write(b)
	Expect(err).NotTo(HaveOccurred())
}
Example #13
0
func (metrics *metrics) processMetric(metric *events.Envelope) {
	switch metric.GetEventType() {
	case events.Envelope_ValueMetric:
		metrics.processValueMetric(metric)
	case events.Envelope_CounterEvent:
		metrics.processCounterEvent(metric)
	case events.Envelope_HttpStartStop:
		metrics.processHTTPStartStop(metric)
	}
}
func getName(envelope *events.Envelope) string {
	switch envelope.GetEventType() {
	case events.Envelope_ValueMetric:
		return envelope.GetOrigin() + "." + envelope.GetValueMetric().GetName()
	case events.Envelope_CounterEvent:
		return envelope.GetOrigin() + "." + envelope.GetCounterEvent().GetName()
	default:
		panic("Unknown event type")
	}
}
func (m *MessageAggregator) handleHTTPStart(envelope *events.Envelope) {
	metrics.BatchIncrementCounter("MessageAggregator.httpStartReceived")

	logging.Debugf(m.logger, "handling HTTP start message for appID: %v", envelope.GetHttpStart().GetApplicationId())
	startEvent := envelope.GetHttpStart()

	requestID := startEvent.RequestId.String()
	event := eventID{requestID: requestID, peerType: startEvent.GetPeerType()}
	m.startEventsByEventID[event] = startEventEntry{startEvent: startEvent, entryTime: time.Now()}
}
Example #16
0
func basicTaggedHttpStartStopMessage(envelope events.Envelope) *events.Envelope {
	ip, _ := localip.LocalIP()

	envelope.Deployment = proto.String("test-deployment")
	envelope.Job = proto.String("test-job")
	envelope.Index = proto.String("2")
	envelope.Ip = proto.String(ip)

	return &envelope
}
Example #17
0
func (m *MessageAggregator) handleHTTPStart(envelope *events.Envelope) {
	m.incrementCounter(&m.httpStartReceivedCount)

	m.logger.Debugf("handling HTTP start message %v", spew.Sprintf("%v", envelope))
	startEvent := envelope.GetHttpStart()

	requestID := startEvent.RequestId.String()
	event := eventID{requestID: requestID, peerType: startEvent.GetPeerType()}
	m.startEventsByEventID[event] = startEventEntry{startEvent: startEvent, entryTime: time.Now()}
}
Example #18
0
func (m *MetricsNozzle) handleMessage(envelope *events.Envelope) {
	if m.config.OriginID != "" {
		if envelope.GetOrigin() == m.config.OriginID {
			fmt.Fprintf(m.stdout, "%v \n", envelope)
		}
	} else {
		fmt.Fprintf(m.stdout, "%v \n", envelope)
	}

}
Example #19
0
func update(values map[string]int64, event *events.Envelope) int64 {

	values[event.GetIp()] = int64(event.GetCounterEvent().GetTotal())

	var sum int64
	for _, v := range values {
		sum = sum + v
	}

	return sum
}
Example #20
0
func (m *MsgLossChart) ProcessEvent(evt *events.Envelope) {
	switch evt.GetCounterEvent().GetName() {
	case "DopplerForwarder.sentMessages":
		m.totalSent = update(m.sentByIP, evt)
	case "tlsListener.receivedMessageCount", "dropsondeListener.receivedMessageCount":
		m.totalReceived = update(m.receivedByIP, evt)
	}
	percent := 100 * ((m.totalSent - m.totalReceived) / m.totalSent)

	m.graph.Percent = int(percent)
}
Example #21
0
func (s *UAAChart) ProcessEvent(event *events.Envelope) {
	switch event.GetValueMetric().GetName() {
	case "audit_service.user_authentication_count":
		s.data[0] = updateAndReturnValue(s.dataByIp[0], event)
	case "audit_service.user_authentication_failure_count":
		s.data[1] = updateAndReturnValue(s.dataByIp[1], event)
	case "audit_service.principal_authentication_failure_count":
		s.data[2] = updateAndReturnValue(s.dataByIp[2], event)
	case "audit_service.user_password_failures":
		s.data[3] = updateAndReturnValue(s.dataByIp[3], event)
	}
}
Example #22
0
func (m *EventMarshaller) Write(message *events.Envelope) {
	messageBytes, err := proto.Marshal(message)
	if err != nil {
		m.logger.Errorf("eventMarshaller: marshal error %v for message %v", err, message)
		metrics.BatchIncrementCounter("dropsondeMarshaller.marshalErrors")
		return
	}

	m.logger.Debugf("eventMarshaller: marshalled message %v", spew.Sprintf("%v", message))
	m.incrementMessageCount(message.GetEventType())
	m.outputWriter.Write(messageBytes)
}
func (sink *ContainerMetricSink) updateMetric(event *events.Envelope) {
	sink.lock.Lock()
	defer sink.lock.Unlock()

	instance := event.GetContainerMetric().GetInstanceIndex()

	oldMetric, ok := sink.metrics[instance]

	if !ok || oldMetric.GetTimestamp() < event.GetTimestamp() {
		sink.metrics[instance] = event
	}
}
func (m *MessageAggregator) handleHTTPStop(envelope *events.Envelope) *events.Envelope {
	if m.emitMetrics {
		metrics.BatchIncrementCounter("MessageAggregator.httpStopReceived")
	}
	atomic.AddUint64(&m.httpStopReceivedCount, 1)

	m.logger.Debugf("handling HTTP stop message %v", spew.Sprintf("%v", envelope))
	stopEvent := envelope.GetHttpStop()

	requestID := stopEvent.RequestId.String()
	event := eventID{requestID: requestID, peerType: stopEvent.GetPeerType()}

	startEventEntry, ok := m.startEventsByEventID[event]
	if !ok {
		m.logger.Warnf("no matching HTTP start message found for %v", event)
		if m.emitMetrics {
			metrics.BatchIncrementCounter("MessageAggregator.httpUnmatchedStopReceived")
		}
		atomic.AddUint64(&m.httpUnmatchedStopReceivedCount, 1)
		return nil
	}

	if m.emitMetrics {
		metrics.BatchIncrementCounter("MessageAggregator.httpStartStopEmitted")
	}
	atomic.AddUint64(&m.httpStartStopEmittedCount, 1)

	delete(m.startEventsByEventID, event)
	startEvent := startEventEntry.startEvent

	return &events.Envelope{
		Origin:    envelope.Origin,
		Timestamp: stopEvent.Timestamp,
		EventType: events.Envelope_HttpStartStop.Enum(),
		HttpStartStop: &events.HttpStartStop{
			StartTimestamp:  startEvent.Timestamp,
			StopTimestamp:   stopEvent.Timestamp,
			RequestId:       startEvent.RequestId,
			PeerType:        startEvent.PeerType,
			Method:          startEvent.Method,
			Uri:             startEvent.Uri,
			RemoteAddress:   startEvent.RemoteAddress,
			UserAgent:       startEvent.UserAgent,
			StatusCode:      stopEvent.StatusCode,
			ContentLength:   stopEvent.ContentLength,
			ParentRequestId: startEvent.ParentRequestId,
			ApplicationId:   stopEvent.ApplicationId,
			InstanceIndex:   startEvent.InstanceIndex,
			InstanceId:      startEvent.InstanceId,
		},
	}
}
Example #25
0
func (s *SyslogSink) sendMessage(messageEnvelope *events.Envelope) bool {
	logMessage := messageEnvelope.GetLogMessage()

	_, err := s.syslogWriter.Write(messagePriorityValue(logMessage), logMessage.GetMessage(), logMessage.GetSourceType(), logMessage.GetSourceInstance(), *logMessage.Timestamp)

	if err != nil {
		s.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", s.drainUrl, err)
		return false
	} else {
		s.Debugf("Syslog Sink %s: Successfully sent data\n", s.drainUrl)
		return true
	}
}
Example #26
0
func ErrorEvent(msg *events.Envelope) Event {
	errorEvent := msg.GetError()

	fields := logrus.Fields{
		"code":  errorEvent.GetCode(),
		"delta": errorEvent.GetSource(),
	}

	return Event{
		Fields: fields,
		Msg:    errorEvent.GetMessage(),
	}
}
Example #27
0
func ValueMetric(msg *events.Envelope) Event {
	valMetric := msg.GetValueMetric()

	fields := logrus.Fields{
		"name":  valMetric.GetName(),
		"unit":  valMetric.GetUnit(),
		"value": valMetric.GetValue(),
	}

	return Event{
		Fields: fields,
		Msg:    "",
	}
}
Example #28
0
func (s *SinkTypeChart) ProcessEvent(event *events.Envelope) {
	switch event.GetValueMetric().GetName() {
	case "messageRouter.numberOfContainerMetricSinks":
		s.data[0] = updateAndReturnValue(s.dataByIp[0], event)
	case "messageRouter.numberOfSyslogSinks":
		s.data[1] = updateAndReturnValue(s.dataByIp[1], event)
	case "messageRouter.numberOfDumpSinks":
		s.data[2] = updateAndReturnValue(s.dataByIp[2], event)
	case "messageRouter.numberOfWebsocketSinks":
		s.data[3] = updateAndReturnValue(s.dataByIp[3], event)
	case "messageRouter.numberOfFirehoseSinks":
		s.data[4] = updateAndReturnValue(s.dataByIp[4], event)
	}
}
Example #29
0
func (t *Tagger) setDefaultTags(envelope *events.Envelope) {
	if envelope.Deployment == nil {
		envelope.Deployment = proto.String(t.deploymentName)
	}
	if envelope.Job == nil {
		envelope.Job = proto.String(t.job)
	}
	if envelope.Index == nil {
		envelope.Index = proto.String(t.index)
	}
	if envelope.Ip == nil {
		envelope.Ip = proto.String(t.ip)
	}
}
Example #30
0
func CounterEvent(msg *events.Envelope) Event {
	counterEvent := msg.GetCounterEvent()

	fields := logrus.Fields{
		"name":  counterEvent.GetName(),
		"delta": counterEvent.GetDelta(),
		"total": counterEvent.GetTotal(),
	}

	return Event{
		Fields: fields,
		Msg:    "",
	}
}