func (m *MessageAggregator) Write(envelope *events.Envelope) {
	// TODO: don't call for every message if throughput becomes a problem
	m.cleanupOrphanedHTTPStart()

	if envelope.EventType == nil {
		m.outputWriter.Write(envelope)
		return
	}
	switch envelope.GetEventType() {
	case events.Envelope_HttpStart:
		m.handleHTTPStart(envelope)
	case events.Envelope_HttpStop:
		startStopMessage := m.handleHTTPStop(envelope)
		if startStopMessage != nil {
			m.outputWriter.Write(startStopMessage)
		}
	case events.Envelope_CounterEvent:
		counterEventMessage := m.handleCounter(envelope)
		m.outputWriter.Write(counterEventMessage)
	default:
		m.incrementCounter(&m.uncategorizedEventCount)
		m.logger.Debugf("passing through message %v", spew.Sprintf("%v", envelope))
		m.outputWriter.Write(envelope)
	}
}
func (m *MessageAggregator) Write(envelope *events.Envelope) {
	// TODO: don't call for every message if throughput becomes a problem
	m.cleanupOrphanedHTTPStart()

	if envelope.EventType == nil {
		metrics.BatchIncrementCounter("MessageAggregator.uncategorizedEvents")
		return
	}

	switch envelope.GetEventType() {
	case events.Envelope_HttpStart:
		m.handleHTTPStart(envelope)
	case events.Envelope_HttpStop:
		startStopMessage := m.handleHTTPStop(envelope)
		if startStopMessage != nil {
			m.outputWriter.Write(startStopMessage)
		}
	case events.Envelope_CounterEvent:
		counterEventMessage := m.handleCounter(envelope)
		m.outputWriter.Write(counterEventMessage)
	default:
		metrics.BatchIncrementCounter("MessageAggregator.uncategorizedEvents")
		m.outputWriter.Write(envelope)
	}
}
func HttpStartStop(msg *events.Envelope) Event {
	httpStartStop := msg.GetHttpStartStop()

	fields := logrus.Fields{
		"origin":            msg.GetOrigin(),
		"cf_app_id":         httpStartStop.GetApplicationId(),
		"content_length":    httpStartStop.GetContentLength(),
		"instance_id":       httpStartStop.GetInstanceId(),
		"instance_index":    httpStartStop.GetInstanceIndex(),
		"method":            httpStartStop.GetMethod(),
		"parent_request_id": httpStartStop.GetParentRequestId(),
		"peer_type":         httpStartStop.GetPeerType(),
		"remote_addr":       httpStartStop.GetRemoteAddress(),
		"request_id":        httpStartStop.GetRequestId(),
		"start_timestamp":   httpStartStop.GetStartTimestamp(),
		"status_code":       httpStartStop.GetStatusCode(),
		"stop_timestamp":    httpStartStop.GetStopTimestamp(),
		"uri":               httpStartStop.GetUri(),
		"user_agent":        httpStartStop.GetUserAgent(),
		"duration_ms":       (((httpStartStop.GetStopTimestamp() - httpStartStop.GetStartTimestamp()) / 1000) / 1000),
	}

	return Event{
		Fields: fields,
		Msg:    "",
		Type:   msg.GetEventType().String(),
	}
}
func dopplerReceivedMessageCount(msg *events.Envelope) bool {
	return msg.GetEventType() == events.Envelope_CounterEvent &&
		(msg.CounterEvent.GetName() == "udpListener.receivedMessageCount" ||
			msg.CounterEvent.GetName() == "tcpListener.receivedMessageCount" ||
			msg.CounterEvent.GetName() == "tlsListener.receivedMessageCount") &&
		msg.GetOrigin() == "DopplerServer"
}
Beispiel #5
0
func (l *logCounter) processEnvelope(env *events.Envelope) {
	if env.GetEventType() != events.Envelope_LogMessage {
		return
	}
	logMsg := env.GetLogMessage()

	msg := string(logMsg.GetMessage())
	if strings.HasPrefix(msg, "mismatched prefix") {
		return
	}
	if !strings.HasPrefix(msg, l.cfg.MessagePrefix) {
		fmt.Printf("mismatched prefix: log message %s did not match prefix: %s\n", string(logMsg.GetMessage()), string(l.cfg.MessagePrefix))
		return
	}

	if len(msg) < sepEnd {
		fmt.Printf("Cannot parse message %s\n", msg)
		return
	}

	id := Identity{
		appID: logMsg.GetAppId(),
		runID: msg[prefixEnd:guidEnd],
	}

	l.counterLock.Lock()
	defer l.counterLock.Unlock()
	counter, ok := l.counters[id]
	if !ok {
		counter = make(map[string]bool)
		l.counters[id] = counter
	}
	counter[msg[sepEnd:]] = true
}
func TranslateDropsondeToLegacyLogMessage(message []byte) ([]byte, error) {
	var receivedEnvelope events.Envelope
	err := proto.Unmarshal(message, &receivedEnvelope)
	if err != nil {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Unable to unmarshal bytes as Envelope: %v", err)
	}

	if receivedEnvelope.GetEventType() != events.Envelope_LogMessage {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope contained %s instead of LogMessage", receivedEnvelope.GetEventType().String())
	}

	logMessage := receivedEnvelope.GetLogMessage()
	if logMessage == nil {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Envelope's LogMessage was nil: %v", receivedEnvelope)
	}

	messageBytes, err := proto.Marshal(
		&logmessage.LogMessage{
			Message:     logMessage.GetMessage(),
			MessageType: (*logmessage.LogMessage_MessageType)(logMessage.MessageType),
			Timestamp:   proto.Int64(logMessage.GetTimestamp()),
			AppId:       proto.String(logMessage.GetAppId()),
			SourceId:    proto.String(logMessage.GetSourceInstance()),
			SourceName:  proto.String(logMessage.GetSourceType()),
		},
	)
	if err != nil {
		return nil, fmt.Errorf("TranslateDropsondeToLegacyLogMessage: Failed marshalling converted dropsonde message: %v", err)
	}

	return messageBytes, nil
}
Beispiel #7
0
func routeEvent(msg *events.Envelope, extraFields map[string]string) {

	eventType := msg.GetEventType()

	if selectedEvents[eventType.String()] {
		var event Event
		switch eventType {
		case events.Envelope_HttpStart:
			event = HttpStart(msg)
		case events.Envelope_HttpStop:
			event = HttpStop(msg)
		case events.Envelope_HttpStartStop:
			event = HttpStartStop(msg)
		case events.Envelope_LogMessage:
			event = LogMessage(msg)
		case events.Envelope_ValueMetric:
			event = ValueMetric(msg)
		case events.Envelope_CounterEvent:
			event = CounterEvent(msg)
		case events.Envelope_Error:
			event = ErrorEvent(msg)
		case events.Envelope_ContainerMetric:
			event = ContainerMetric(msg)
		}

		event.AnnotateWithAppData()
		event.AnnotateWithMetaData(extraFields)
		event.ShipEvent()

		mutex.Lock()
		selectedEventsCount[eventType.String()]++
		mutex.Unlock()
	}
}
func listenForLogs() {
	for {
		buffer := make([]byte, 1024)
		n, _, err := udpConn.ReadFrom(buffer)
		if err != nil {
			return
		}

		if n == 0 {
			panic("Received empty packet")
		}
		envelope := new(events.Envelope)
		err = proto.Unmarshal(buffer[0:n], envelope)
		if err != nil {
			panic(err)
		}

		if envelope.GetEventType() == events.Envelope_LogMessage {
			logLock.Lock()
			logMessages = append(logMessages, envelope.GetLogMessage())
			logLock.Unlock()

		}
	}
}
func (d *DopplerForwarder) networkWrite(message *events.Envelope) {
	client, err := d.clientPool.RandomClient()
	if err != nil {
		d.logger.Errord(map[string]interface{}{
			"error": err.Error(),
		}, "DopplerForwarder: can't forward message")
		return
	}

	messageBytes, err := proto.Marshal(message)
	if err != nil {
		d.logger.Errorf("DopplerForwarder: marshal error %v", err)
		metrics.BatchIncrementCounter("dropsondeMarshaller.marshalErrors")
		return
	}

	switch client.Scheme() {
	case "udp":
		signedMessage := signature.SignMessage(messageBytes, d.sharedSecret)
		bytesWritten, err := client.Write(signedMessage)
		if err != nil {
			metrics.BatchIncrementCounter("udp.sendErrorCount")
			d.logger.Debugd(map[string]interface{}{
				"scheme":  client.Scheme(),
				"address": client.Address(),
			}, "Error writing legacy message")
			return
		}
		metrics.BatchIncrementCounter("udp.sentMessageCount")
		metrics.BatchAddCounter("udp.sentByteCount", uint64(bytesWritten))
	case "tls":
		var bytesWritten int
		err = binary.Write(client, binary.LittleEndian, uint32(len(messageBytes)))
		if err == nil {
			bytesWritten, err = client.Write(messageBytes)
		}

		if err != nil {
			metrics.BatchIncrementCounter("tls.retryCount")
			d.inputChan <- message
			client.Close()

			d.logger.Errord(map[string]interface{}{
				"scheme":  client.Scheme(),
				"address": client.Address(),
				"error":   err.Error(),
			}, "DopplerForwarder: streaming error")
			return
		}
		metrics.BatchIncrementCounter("tls.sentMessageCount")
		metrics.BatchAddCounter("tls.sentByteCount", uint64(bytesWritten+4))
	default:
		d.logger.Errorf("DopplerForwarder: unknown protocol, %s for %s", client.Scheme(), client.Address())
		return
	}

	d.incrementMessageCount(message.GetEventType())
	metrics.BatchIncrementCounter("DopplerForwarder.sentMessages")
}
func getValue(envelope *events.Envelope) float64 {
	switch envelope.GetEventType() {
	case events.Envelope_ValueMetric:
		return envelope.GetValueMetric().GetValue()
	case events.Envelope_CounterEvent:
		return float64(envelope.GetCounterEvent().GetTotal())
	default:
		panic("Unknown event type")
	}
}
func getName(envelope *events.Envelope) string {
	switch envelope.GetEventType() {
	case events.Envelope_ValueMetric:
		return envelope.GetOrigin() + "." + envelope.GetValueMetric().GetName()
	case events.Envelope_CounterEvent:
		return envelope.GetOrigin() + "." + envelope.GetCounterEvent().GetName()
	default:
		panic("Unknown event type")
	}
}
Beispiel #12
0
func (metrics *metrics) processMetric(metric *events.Envelope) {
	switch metric.GetEventType() {
	case events.Envelope_ValueMetric:
		metrics.processValueMetric(metric)
	case events.Envelope_CounterEvent:
		metrics.processCounterEvent(metric)
	case events.Envelope_HttpStartStop:
		metrics.processHTTPStartStop(metric)
	}
}
Beispiel #13
0
func (e *Event) AnnotateWithEnveloppeData(msg *events.Envelope) {
	e.Fields["origin"] = msg.GetOrigin()
	e.Fields["deployment"] = msg.GetDeployment()
	e.Fields["ip"] = msg.GetIp()
	e.Fields["job"] = msg.GetJob()
	e.Fields["index"] = msg.GetIndex()
	//e.Fields["tags"] = msg.GetTags()
	e.Type = msg.GetEventType().String()

}
func checkValueMetric(receivedEnvelope *events.Envelope, origin string, name string, value float64, unit string) {
	Expect(receivedEnvelope.GetEventType()).To(Equal(events.Envelope_ValueMetric))
	Expect(receivedEnvelope.GetOrigin()).To(Equal(origin))

	vm := receivedEnvelope.GetValueMetric()
	Expect(vm.GetName()).To(Equal(name))
	Expect(vm.GetValue()).To(BeNumerically("==", value))
	Expect(vm.GetUnit()).To(Equal(unit))

}
func (f *fakeWriter) Write(message *events.Envelope) {
	if message.GetEventType() == events.Envelope_ValueMetric {

		switch message.GetValueMetric().GetName() {
		case "Uptime":
			atomic.StoreUint64(&f.lastUptime, uint64(message.GetValueMetric().GetValue()))
		case "LinuxFileDescriptor":
			atomic.StoreUint64(&f.openFileDescriptors, uint64(message.GetValueMetric().GetValue()))
		}
	}
}
Beispiel #16
0
func (m *MsgLossChart) ForChart(event *events.Envelope) bool {
	if event.GetEventType() != events.Envelope_CounterEvent {
		return false
	}
	if !contains(event.GetOrigin(), m.validOrigins) {
		return false
	}
	if !contains(event.GetCounterEvent().GetName(), m.validMetricNames) {
		return false
	}
	return true
}
Beispiel #17
0
func (m *EventMarshaller) Write(message *events.Envelope) {
	messageBytes, err := proto.Marshal(message)
	if err != nil {
		m.logger.Errorf("eventMarshaller: marshal error %v for message %v", err, message)
		metrics.BatchIncrementCounter("dropsondeMarshaller.marshalErrors")
		return
	}

	m.logger.Debugf("eventMarshaller: marshalled message %v", spew.Sprintf("%v", message))
	m.incrementMessageCount(message.GetEventType())
	m.outputWriter.Write(messageBytes)
}
Beispiel #18
0
func (s *SinkTypeChart) ForChart(event *events.Envelope) bool {
	if event.GetEventType() != events.Envelope_ValueMetric {
		return false
	}
	if !contains(event.GetOrigin(), s.validOrigins) {
		return false
	}
	if !contains(event.GetValueMetric().GetName(), s.validMetricNames) {
		return false
	}

	//	s.cfUI.Say("%f ", event.GetValueMetric().GetValue())
	return true
}
func (c *Client) AddMetric(envelope *events.Envelope) {
	c.totalMessagesReceived++
	if envelope.GetEventType() != events.Envelope_ValueMetric && envelope.GetEventType() != events.Envelope_CounterEvent {
		return
	}
	metric := poster.Metric{
		Value:     getValue(envelope),
		Timestamp: envelope.GetTimestamp() / int64(time.Second),
		Metric:    c.prefix + getName(envelope),
		Tags:      getTags(envelope),
	}

	c.metrics = append(c.metrics, metric)
}
func ErrorEvent(msg *events.Envelope) Event {
	errorEvent := msg.GetError()

	fields := logrus.Fields{
		"origin": msg.GetOrigin(),
		"code":   errorEvent.GetCode(),
		"delta":  errorEvent.GetSource(),
	}

	return Event{
		Fields: fields,
		Msg:    errorEvent.GetMessage(),
		Type:   msg.GetEventType().String(),
	}
}
Beispiel #21
0
func updateAndReturnValue(values map[string]int, event *events.Envelope) int {

	switch event.GetEventType() {
	case events.Envelope_CounterEvent:
		values[event.GetIp()] = int(event.GetCounterEvent().GetTotal())
	case events.Envelope_ValueMetric:
		values[event.GetIp()] = int(event.GetValueMetric().GetValue())
	}

	sum := 0
	for _, v := range values {
		sum += v
	}

	return sum
}
func ValueMetric(msg *events.Envelope) Event {
	valMetric := msg.GetValueMetric()

	fields := logrus.Fields{
		"origin": msg.GetOrigin(),
		"name":   valMetric.GetName(),
		"unit":   valMetric.GetUnit(),
		"value":  valMetric.GetValue(),
	}

	return Event{
		Fields: fields,
		Msg:    "",
		Type:   msg.GetEventType().String(),
	}
}
func CounterEvent(msg *events.Envelope) Event {
	counterEvent := msg.GetCounterEvent()

	fields := logrus.Fields{
		"origin": msg.GetOrigin(),
		"name":   counterEvent.GetName(),
		"delta":  counterEvent.GetDelta(),
		"total":  counterEvent.GetTotal(),
	}

	return Event{
		Fields: fields,
		Msg:    "",
		Type:   msg.GetEventType().String(),
	}
}
func routeEvent(msg *events.Envelope, extraFields map[string]string, filters filters.LogFilters) {

	eventType := msg.GetEventType()

	if selectedEvents[eventType.String()] {
		var event Event
		switch eventType {
		case events.Envelope_HttpStart:
			event = HttpStart(msg)
		case events.Envelope_HttpStop:
			event = HttpStop(msg)
		case events.Envelope_HttpStartStop:
			event = HttpStartStop(msg)
		case events.Envelope_LogMessage:
			event = LogMessage(msg)
		case events.Envelope_ValueMetric:
			event = ValueMetric(msg)
		case events.Envelope_CounterEvent:
			event = CounterEvent(msg)
		case events.Envelope_Error:
			event = ErrorEvent(msg)
		case events.Envelope_ContainerMetric:
			event = ContainerMetric(msg)
		}

		event.AnnotateWithAppData()
		event.AnnotateWithMetaData(extraFields)

		if event.ApplyFilters(filters) {
			// Compose the key to be used in Redis
			orgId := event.Fields["cf_org_id"].(string)
			orgEnabledKey := orgId + ":enabled"

			// Get the "enabled" flag to know if this organization will keep sending logs to ELK
			orgEnabled, _ := redis.Bool(redisConn.Do("HGET", redisHash, orgEnabledKey))
			if orgEnabled {
				// Increment the value of the organization key to add the amount of bytes of the log
				value, err := redisConn.Do("HINCRBY", redisHash, orgId, len(event.Msg))
				if err != nil || value == nil {
					panic("Redis server could not be contacted!")
				}
				// Send the event (log message)
				event.ShipEvent()
			}
		}
	}
}
func valid(env *events.Envelope) bool {
	switch env.GetEventType() {
	case events.Envelope_HttpStartStop:
		return env.GetHttpStartStop() != nil
	case events.Envelope_LogMessage:
		return env.GetLogMessage() != nil
	case events.Envelope_ValueMetric:
		return env.GetValueMetric() != nil
	case events.Envelope_CounterEvent:
		return env.GetCounterEvent() != nil
	case events.Envelope_Error:
		return env.GetError() != nil
	case events.Envelope_ContainerMetric:
		return env.GetContainerMetric() != nil
	}
	return true
}
func listenForEvents(origin []string) {
	for {
		buffer := make([]byte, 1024)
		n, _, err := udpListener.ReadFrom(buffer)
		if err != nil {
			return
		}

		if n == 0 {
			panic("Received empty packet")
		}
		envelope := new(events.Envelope)
		err = proto.Unmarshal(buffer[0:n], envelope)
		if err != nil {
			panic(err)
		}

		var eventId = envelope.GetEventType().String()

		tracker := eventTracker{eventType: eventId}

		switch envelope.GetEventType() {
		case events.Envelope_HttpStart:
			tracker.name = envelope.GetHttpStart().GetPeerType().String()
		case events.Envelope_HttpStop:
			tracker.name = envelope.GetHttpStop().GetPeerType().String()
		case events.Envelope_ValueMetric:
			tracker.name = envelope.GetValueMetric().GetName()
		case events.Envelope_CounterEvent:
			tracker.name = envelope.GetCounterEvent().GetName()
		default:
			panic("Unexpected message type")

		}

		if envelope.GetOrigin() != strings.Join(origin, "/") {
			panic("origin not as expected")
		}

		func() {
			lock.Lock()
			defer lock.Unlock()
			receivedEvents = append(receivedEvents, tracker)
		}()
	}
}
func LogMessage(msg *events.Envelope) Event {
	logMessage := msg.GetLogMessage()

	fields := logrus.Fields{
		"origin":          msg.GetOrigin(),
		"cf_app_id":       logMessage.GetAppId(),
		"timestamp":       logMessage.GetTimestamp(),
		"source_type":     logMessage.GetSourceType(),
		"message_type":    logMessage.GetMessageType().String(),
		"source_instance": logMessage.GetSourceInstance(),
	}

	return Event{
		Fields: fields,
		Msg:    string(logMessage.GetMessage()),
		Type:   msg.GetEventType().String(),
	}
}
func ContainerMetric(msg *events.Envelope) Event {
	containerMetric := msg.GetContainerMetric()

	fields := logrus.Fields{
		"origin":         msg.GetOrigin(),
		"cf_app_id":      containerMetric.GetApplicationId(),
		"cpu_percentage": containerMetric.GetCpuPercentage(),
		"disk_bytes":     containerMetric.GetDiskBytes(),
		"instance_index": containerMetric.GetInstanceIndex(),
		"memory_bytes":   containerMetric.GetMemoryBytes(),
	}

	return Event{
		Fields: fields,
		Msg:    "",
		Type:   msg.GetEventType().String(),
	}
}
func (m *EventMarshaller) Write(envelope *events.Envelope) {
	writer := m.writer()
	if writer == nil {
		m.logger.Warn("EventMarshaller: Write called while byteWriter is nil")
		return
	}

	envelopeBytes, err := proto.Marshal(envelope)
	if err != nil {
		m.logger.Errorf("marshalling error: %v", err)
		m.batcher.BatchIncrementCounter("dropsondeMarshaller.marshalErrors")
		return
	}

	chainer := m.batcher.BatchCounter("dropsondeMarshaller.sentEnvelopes").
		SetTag("event_type", envelope.GetEventType().String())

	writer.Write(envelopeBytes, chainer)
}
func HttpStop(msg *events.Envelope) Event {
	httpStop := msg.GetHttpStop()

	fields := logrus.Fields{
		"origin":         msg.GetOrigin(),
		"cf_app_id":      httpStop.GetApplicationId(),
		"content_length": httpStop.GetContentLength(),
		"peer_type":      httpStop.GetPeerType(),
		"request_id":     httpStop.GetRequestId(),
		"status_code":    httpStop.GetStatusCode(),
		"timestamp":      httpStop.GetTimestamp(),
		"uri":            httpStop.GetUri(),
	}

	return Event{
		Fields: fields,
		Msg:    "",
		Type:   msg.GetEventType().String(),
	}
}