func (r *TruncatingBuffer) Run() {
	for msg := range r.inputChannel {
		r.lock.Lock()
		select {
		case r.outputChannel <- msg:
		default:
			messageCount := len(r.outputChannel)
			r.droppedMessageCount += int64(messageCount)
			r.outputChannel = make(chan *events.Envelope, cap(r.outputChannel))
			appId := envelope_extensions.GetAppId(msg)
			lm := generateLogMessage(fmt.Sprintf("Log message output too high. We've dropped %d messages", messageCount), appId)

			env, err := emitter.Wrap(lm, r.dropsondeOrigin)
			if err == nil {
				r.outputChannel <- env
			} else {
				r.logger.Warnf("Error marshalling message: %v", err)
			}

			r.outputChannel <- msg

			if r.logger != nil {
				r.logger.Warn(fmt.Sprintf("TB: Output channel too full. Dropped %d messages for app %s.", messageCount, appId))
			}
		}
		r.lock.Unlock()
	}
	close(r.outputChannel)
}
func (e *EventWriter) Emit(event events.Event) error {
	envelope, err := emitter.Wrap(event, e.origin)
	if err != nil {
		return err
	}

	return e.EmitEnvelope(envelope)
}
func (r *TruncatingBuffer) emitMessage(outputChannel chan *events.Envelope, event events.Event) {
	env, err := emitter.Wrap(event, r.context.DropsondeOrigin())
	if err == nil {
		outputChannel <- env
	} else {
		r.logger.Warnf("Error marshalling message: %v", err)
	}
}
Exemple #4
0
func (sinkManager *SinkManager) SendSyslogErrorToLoggregator(errorMsg string, appId string) {
	sinkManager.logger.Warn(errorMsg)

	logMessage := factories.NewLogMessage(events.LogMessage_ERR, errorMsg, appId, "LGR")
	envelope, err := emitter.Wrap(logMessage, sinkManager.dropsondeOrigin)
	if err != nil {
		sinkManager.logger.Warnf("Error marshalling message: %v", err)
		return
	}

	sinkManager.errorChannel <- envelope
}
Exemple #5
0
func DropsondeLogMessage(messageString string, appId string) []byte {
	currentTime := time.Now()
	logMessage := &events.LogMessage{
		Message:     []byte(messageString),
		MessageType: events.LogMessage_ERR.Enum(),
		Timestamp:   proto.Int64(currentTime.UnixNano()),
		SourceType:  proto.String("DOP"),
		AppId:       &appId,
	}

	envelope, _ := emitter.Wrap(logMessage, "doppler")

	msg, _ := proto.Marshal(envelope)
	return msg
}
Exemple #6
0
func main() {
	flag.Parse()

	la, err := net.ResolveUDPAddr("udp", *destination)
	if err != nil {
		log.Fatalf("Error resolving loggregator address %s, %s", *destination, err)
	}

	//    connection, err := net.DialUDP("udp", nil, la)
	connection, err := net.ListenPacket("udp4", "")
	if err != nil {
		log.Fatalf("Error opening udp stuff")
	}

	msg := make([]byte, *size, *size)
	envelope, err := emitter.Wrap(&events.LogMessage{
		Message:     msg,
		MessageType: events.LogMessage_OUT.Enum(),
		Timestamp:   proto.Int64(time.Now().UnixNano()),
	}, "origin")
	if err != nil {
		log.Fatal(err.Error())
	}

	buf, err := proto.Marshal(envelope)
	if err != nil {
		log.Fatal(err.Error())
	}

	finalBytes := signature.SignMessage(buf, []byte(*secret))

	//	println("marshal time", t2.Sub(t1).String())

	var i int
	d := time.Duration(*duration)
	time.AfterFunc(d*time.Second, func() {
		fmt.Printf("%d, %d, %d, ", *duration, len(finalBytes), i)
		connection.Close()
		os.Exit(0)
	})

	t := time.NewTicker(time.Second / time.Duration(*rate))
	for {
		<-t.C
		connection.WriteTo(finalBytes, la)
		i++
	}
}
Exemple #7
0
func SendEventTCP(event events.Event, conn net.Conn) error {
	envelope, err := emitter.Wrap(event, "origin")
	Expect(err).NotTo(HaveOccurred())

	bytes, err := proto.Marshal(envelope)
	if err != nil {
		return err
	}

	err = binary.Write(conn, binary.LittleEndian, uint32(len(bytes)))
	if err != nil {
		return err
	}

	_, err = conn.Write(bytes)
	return err
}
Exemple #8
0
func (w *Writer) droppedLogMessage() []byte {
	droppedMessages := atomic.LoadUint64(&w.droppedMessages)
	logMessage := &events.LogMessage{
		Message:     []byte(fmt.Sprintf("Dropped %d message(s) from MetronAgent to Doppler", droppedMessages)),
		MessageType: events.LogMessage_ERR.Enum(),
		AppId:       proto.String(envelope_extensions.SystemAppId),
		Timestamp:   proto.Int64(time.Now().UnixNano()),
	}
	env, err := emitter.Wrap(logMessage, "MetronAgent")
	if err != nil {
		w.logger.Fatalf("Failed to emitter.Wrap a log message: %s", err)
	}
	marshaled, err := proto.Marshal(env)
	if err != nil {
		w.logger.Fatalf("Failed to marshal generated dropped log message: %s", err)
	}
	prefixedBytes, err := w.prefixMessage(marshaled)
	if err != nil {
		w.logger.Fatalf("Failed to prefix dropped log message: %s", err)
	}

	return prefixedBytes
}
Exemple #9
0
func MarshalEvent(event events.Event, secret string) []byte {
	envelope, _ := emitter.Wrap(event, "origin")
	envelopeBytes := MarshalProtoBuf(envelope)

	return signature.SignMessage(envelopeBytes, []byte(secret))
}
	Describe("SendTo", func() {
		It("sends to all known sinks", func() {
			sink1 := &channelSink{appId: "myApp",
				identifier: "myAppChan1",
				done:       make(chan struct{}),
			}
			sink2 := &channelSink{appId: "myApp",
				identifier: "myAppChan2",
				done:       make(chan struct{}),
			}

			sinkManager.RegisterSink(sink1)
			sinkManager.RegisterSink(sink2)

			expectedMessageString := "Some Data"
			expectedMessage, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, expectedMessageString, "myApp", "App"), "origin")
			go sinkManager.SendTo("myApp", expectedMessage)

			Eventually(sink1.Received).Should(HaveLen(1))
			Eventually(sink2.Received).Should(HaveLen(1))
			Expect(sink1.Received()[0]).To(Equal(expectedMessage))
			Expect(sink2.Received()[0]).To(Equal(expectedMessage))
		})

		It("only sends to sinks that match the appID", func(done Done) {
			sink1 := &channelSink{appId: "myApp1",
				identifier: "myAppChan1",
				done:       make(chan struct{}),
			}
			sink2 := &channelSink{appId: "myApp2",
				identifier: "myAppChan2",
			Expect(websocketSink.ShouldReceiveErrors()).To(BeTrue())
		})
	})

	Describe("Run", func() {
		var inputChan chan *events.Envelope

		BeforeEach(func() {
			inputChan = make(chan *events.Envelope, 10)
		})

		It("forwards messages", func(done Done) {
			defer close(done)
			go websocketSink.Run(inputChan)

			message, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "hello world", "appId", "App"), "origin")
			messageBytes, _ := proto.Marshal(message)

			inputChan <- message
			Eventually(fakeWebsocket.ReadMessages).Should(HaveLen(1))
			Expect(fakeWebsocket.ReadMessages()[0]).To(Equal(messageBytes))

			messageTwo, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "goodbye world", "appId", "App"), "origin")
			messageTwoBytes, _ := proto.Marshal(messageTwo)
			inputChan <- messageTwo
			Eventually(fakeWebsocket.ReadMessages).Should(HaveLen(2))
			Expect(fakeWebsocket.ReadMessages()[1]).To(Equal(messageTwoBytes))
		})

		It("sets write deadline", func() {
			go websocketSink.Run(inputChan)
		goRoutineSpawned.Wait()
	})

	AfterEach(func() {
		sinkManager.Stop()
		TestMessageRouter.Stop()
		TestWebsocketServer.Stop()

		services.Wait()
	})

	It("dumps all messages for an app user", func() {
		expectedFirstMessageString := "Some data 1"
		lm := factories.NewLogMessage(events.LogMessage_OUT, expectedFirstMessageString, "myOtherApp", "APP")
		env1, _ := emitter.Wrap(lm, "ORIGIN")

		expectedSecondMessageString := "Some data 2"
		lm = factories.NewLogMessage(events.LogMessage_OUT, expectedSecondMessageString, "myOtherApp", "APP")
		env2, _ := emitter.Wrap(lm, "ORIGIN")

		dataReadChannel <- env1
		dataReadChannel <- env2

		receivedChan := make(chan []byte, 2)
		_, stopKeepAlive, droppedChannel := AddWSSink(receivedChan, serverPort, "/apps/myOtherApp/recentlogs")

		Eventually(droppedChannel).Should(Receive())

		var firstMarshalledEnvelope, secondMarshalledEnvelope []byte
		Eventually(receivedChan).Should(Receive(&firstMarshalledEnvelope))
	})

	Describe("failed connections", func() {
		It("fails without an appId", func() {
			_, connectionDropped = AddWSSink(wsReceivedChan, fmt.Sprintf("ws://%s/apps//stream", apiEndpoint))
			Expect(connectionDropped).To(BeClosed())
		})

		It("fails with bad path", func() {
			_, connectionDropped = AddWSSink(wsReceivedChan, fmt.Sprintf("ws://%s/apps/my-app/junk", apiEndpoint))
			Expect(connectionDropped).To(BeClosed())
		})
	})

	It("dumps buffer data to the websocket client with /recentlogs", func(done Done) {
		lm, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "my message", appId, "App"), "origin")
		sinkManager.SendTo(appId, lm)

		AddWSSink(wsReceivedChan, fmt.Sprintf("ws://%s/apps/%s/recentlogs", apiEndpoint, appId))

		rlm, err := receiveEnvelope(wsReceivedChan)
		Expect(err).NotTo(HaveOccurred())
		Expect(rlm.GetLogMessage().GetMessage()).To(Equal(lm.GetLogMessage().GetMessage()))
		close(done)
	})

	It("dumps container metric data to the websocket client with /containermetrics", func(done Done) {
		cm := factories.NewContainerMetric(appId, 0, 42.42, 1234, 123412341234)
		envelope, _ := emitter.Wrap(cm, "origin")
		sinkManager.SendTo(appId, envelope)
	"github.com/cloudfoundry/dropsonde/factories"
	"github.com/cloudfoundry/sonde-go/events"

	"github.com/cloudfoundry/dropsonde/emitter"
	"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Truncating Buffer", func() {
	It("works like a channel", func() {
		inMessageChan := make(chan *events.Envelope)
		buffer := truncatingbuffer.NewTruncatingBuffer(inMessageChan, 2, loggertesthelper.Logger(), "dropsonde-origin")
		go buffer.Run()

		logMessage1, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "message 1", "appId", "App"), "origin")
		inMessageChan <- logMessage1
		readMessage := <-buffer.GetOutputChannel()
		Expect(readMessage.GetLogMessage().GetMessage()).To(ContainSubstring("message 1"))

		logMessage2, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "message 2", "appId", "App"), "origin")

		inMessageChan <- logMessage2
		readMessage2 := <-buffer.GetOutputChannel()
		Expect(readMessage2.GetLogMessage().GetMessage()).To(ContainSubstring("message 2"))

	})

	It("works like a truncating channel", func() {
		inMessageChan := make(chan *events.Envelope)
		buffer := truncatingbuffer.NewTruncatingBuffer(inMessageChan, 2, loggertesthelper.Logger(), "dropsonde-origin")
			defer conn.Close()

			Expect(tlsListener.Start).Should(Panic())
		})

		It("panics if you start after a stop", func() {
			conn := openTLSConnection(tlsListener.Address())
			defer conn.Close()

			tlsListener.Stop()
			Expect(tlsListener.Start).Should(Panic())
		})

		It("fails to send message after listener has been stopped", func() {
			logMessage := factories.NewLogMessage(events.LogMessage_OUT, "some message", "appId", "source")
			envelope, _ := emitter.Wrap(logMessage, "origin")
			conn := openTLSConnection(tlsListener.Address())

			err := send(conn, envelope)
			Expect(err).ToNot(HaveOccurred())

			tlsListener.Stop()

			Eventually(func() error {
				return send(conn, envelope)
			}).Should(HaveOccurred())

			conn.Close()
		})
	})
})
		errorHandler          func(string, string, string)
		inputChan             chan *events.Envelope
		bufferSize            uint
		dialer                *net.Dialer
	)

	BeforeEach(func() {
		syslogSinkRunFinished = make(chan bool)
		sysLogger = NewSyslogWriterRecorder()
		errorChannel = make(chan *events.Envelope, 10)
		inputChan = make(chan *events.Envelope)
		dialer = &net.Dialer{}

		errorHandler = func(errorMsg string, appId string, drainUrl string) {
			logMessage := factories.NewLogMessage(events.LogMessage_ERR, errorMsg, appId, "LGR")
			envelope, _ := emitter.Wrap(logMessage, "dropsonde-origin")

			select {
			case errorChannel <- envelope:
			default:
			}
		}

		bufferSize = 100
	})

	JustBeforeEach(func() {
		syslogSink = syslog.NewSyslogSink("appId", "syslog://using-fake", loggertesthelper.Logger(), bufferSize, sysLogger, errorHandler, "dropsonde-origin")
	})

	Context("when remote syslog server is down", func() {
func sendLogMessages(message string, inMessageChan chan<- *events.Envelope) {
	logMessage1, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, message, "appId", "App"), "origin")
	inMessageChan <- logMessage1
}
type unknownEvent struct{}

func (*unknownEvent) ProtoMessage() {}

var _ = Describe("EventFormatter", func() {
	Describe("wrap", func() {
		var origin string

		BeforeEach(func() {
			origin = "testEventFormatter/42"
		})

		It("works with dropsonde status (Heartbeat) events", func() {
			statusEvent := &events.Heartbeat{SentCount: proto.Uint64(1), ErrorCount: proto.Uint64(0)}
			envelope, _ := emitter.Wrap(statusEvent, origin)
			Expect(envelope.GetEventType()).To(Equal(events.Envelope_Heartbeat))
			Expect(envelope.GetHeartbeat()).To(Equal(statusEvent))
		})

		It("works with HttpStart events", func() {
			id, _ := uuid.NewV4()
			testEvent := &events.HttpStart{RequestId: factories.NewUUID(id)}

			envelope, _ := emitter.Wrap(testEvent, origin)
			Expect(envelope.GetEventType()).To(Equal(events.Envelope_HttpStart))
			Expect(envelope.GetHttpStart()).To(Equal(testEvent))
		})

		It("works with HttpStop events", func() {
			id, _ := uuid.NewV4()
					for i := 0; i < runCount; i++ {
						sendLogMessages(fmt.Sprintf("message %d", i), inMessageChan)
					}
					<-readDone
				})
			}, 100)
		})

		Context("lossy", func() {

			BeforeEach(func() {
				bufferSize = 100
			})

			var send = func(count int, delay time.Duration) {
				msg, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "message", "appId", "App"), "origin")
				for i := 0; i < count; i++ {
					inMessageChan <- msg
					time.Sleep(delay)
				}
			}

			var receive = func(count int, delay time.Duration) (totalLost uint64) {
				timeout := time.NewTimer(time.Millisecond)
				for i := 0; i < count; i++ {
					msgs := buffer.GetOutputChannel()
					var msg *events.Envelope
					timeout.Reset(time.Millisecond)
					select {
					case msg = <-msgs:
					case <-timeout.C:
Exemple #20
0
			ws, _ = AddWSSink(receiveChan, "4567", "/firehose/hose-subcription-a")
		})

		AfterEach(func() {
			receiveChan = nil
			ws.Close()
		})

		It("listens for dropsonde log message on TLS port", func() {

			message := "my-random-tls-message"
			guid, _ := uuid.NewV4()
			appID := guid.String()

			logMessage := factories.NewLogMessage(events.LogMessage_OUT, message, appID, "APP")
			envelope, _ := emitter.Wrap(logMessage, "origin")

			encoder.Encode(envelope)

			receivedMessageBytes := []byte{}
			Eventually(receiveChan).Should(Receive(&receivedMessageBytes))

			receivedMessage := DecodeProtoBufLogMessage(receivedMessageBytes)
			Expect(receivedMessage.GetAppId()).To(Equal(appID))
			Expect(string(receivedMessage.GetMessage())).To(Equal(message))

		})

		It("listens for dropsonde counter event on TLS port", func() {

			counterEvent := factories.NewCounterEvent("my-counter", 1)
func (f *fakeSink) UpdateDroppedMessageCount(messageCount int64) {}

var _ = Describe("FirehoseGroup", func() {
	It("sends message to all registered sinks", func() {
		receiveChan1 := make(chan *events.Envelope, 10)
		receiveChan2 := make(chan *events.Envelope, 10)

		sink1 := fakeSink{appId: "firehose-a", sinkId: "sink-a"}
		sink2 := fakeSink{appId: "firehose-a", sinkId: "sink-b"}

		group := firehose_group.NewFirehoseGroup()

		group.AddSink(&sink1, receiveChan1)
		group.AddSink(&sink2, receiveChan2)

		msg, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "test message", "234", "App"), "origin")
		group.BroadcastMessage(msg)

		var nextChannelToReceive chan *events.Envelope
		var rmsg *events.Envelope
		select {
		case rmsg = <-receiveChan1:
			nextChannelToReceive = receiveChan2
		case rmsg = <-receiveChan2:
			nextChannelToReceive = receiveChan1
		}

		Expect(rmsg).To(Equal(msg))

		group.BroadcastMessage(msg)
		Expect(nextChannelToReceive).To(Receive(&msg))
				dialer := &net.Dialer{}
				httpsWriter, err := syslogwriter.NewHttpsWriter(url, appId, true, dialer, 0)
				Expect(err).ToNot(HaveOccurred())

				errorHandler := func(errorMsg string, appId string, drainUrl string) {}

				syslogSink := syslog.NewSyslogSink(appId, server.URL, loggertesthelper.Logger(), bufferSize, httpsWriter, errorHandler, "dropsonde-origin")
				inputChan := make(chan *events.Envelope)

				defer syslogSink.Disconnect()
				go syslogSink.Run(inputChan)

				for i := 0; i < int(bufferSize); i++ {
					msg := fmt.Sprintf("message number %v", i)
					logMessage, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, msg, appId, "App"), "origin")

					inputChan <- logMessage
				}
				close(inputChan)

				Eventually(func() int64 {
					return atomic.LoadInt64(&requests)
				}).Should(BeEquivalentTo(bufferSize))

				// We ignore the difference in timestamps for the 0th iteration because our exponential backoff
				// strategy starts of with a difference of 1 ms
				var diff, prevDiff int64
				for i := 1; i < len(timestampsInMillis)-1; i++ {
					diff = timestampsInMillis[i+1] - timestampsInMillis[i]
					Expect(diff).To(BeNumerically(">", 2*prevDiff))
	It("returns correct path for firehose", func() {
		dopplerEndpoint := doppler_endpoint.NewDopplerEndpoint("firehose", "subscription-123", true)
		Expect(dopplerEndpoint.GetPath()).To(Equal("/firehose/subscription-123"))
	})

	It("returns correct path for recentlogs", func() {
		dopplerEndpoint := doppler_endpoint.NewDopplerEndpoint("recentlogs", "abc123", true)
		Expect(dopplerEndpoint.GetPath()).To(Equal("/apps/abc123/recentlogs"))
	})
})

var _ = Describe("ContainerMetricsHandler", func() {
	It("removes duplicate app container metrics", func() {
		messagesChan := make(chan []byte, 2)

		env1, _ := emitter.Wrap(factories.NewContainerMetric("1", 1, 123, 123, 123), "origin")
		env1.Timestamp = proto.Int64(10000)

		env2, _ := emitter.Wrap(factories.NewContainerMetric("1", 1, 123, 123, 123), "origin")
		env2.Timestamp = proto.Int64(20000)

		bytes1, _ := proto.Marshal(env1)
		bytes2, _ := proto.Marshal(env2)

		messagesChan <- bytes2
		messagesChan <- bytes1
		close(messagesChan)

		outputChan := doppler_endpoint.DeDupe(messagesChan)

		Expect(outputChan).To(HaveLen(1))
)

var _ = Describe("Dump Sink", func() {
	It("works with one message", func() {

		testDump := dump.NewDumpSink("myApp", 1, loggertesthelper.Logger(), time.Second)

		dumpRunnerDone := make(chan struct{})
		inputChan := make(chan *events.Envelope)

		go func() {
			testDump.Run(inputChan)
			close(dumpRunnerDone)
		}()

		logMessage, _ := emitter.Wrap(factories.NewLogMessage(events.LogMessage_OUT, "hi", "appId", "App"), "origin")
		inputChan <- logMessage

		close(inputChan)
		<-dumpRunnerDone

		data := testDump.Dump()
		assert.Equal(GinkgoT(), len(data), 1)
		Expect(string(data[0].GetLogMessage().GetMessage())).To(Equal("hi"))
	})

	It("works with two messages", func() {

		testDump := dump.NewDumpSink("myApp", 2, loggertesthelper.Logger(), time.Second)

		dumpRunnerDone := make(chan struct{})