func NewHttpStart(req *http.Request, peerType events.PeerType, requestId *uuid.UUID) *events.HttpStart {
httpStart := &events.HttpStart{
Timestamp: proto.Int64(time.Now().UnixNano()),
RequestId: NewUUID(requestId),
PeerType: &peerType,
Method: events.HttpStart_Method(events.HttpStart_Method_value[req.Method]).Enum(),
Uri: proto.String(fmt.Sprintf("%s%s", req.Host, req.URL.Path)),
RemoteAddress: proto.String(req.RemoteAddr),
UserAgent: proto.String(req.UserAgent()),
}
if applicationId, err := uuid.ParseHex(req.Header.Get("X-CF-ApplicationID")); err == nil {
httpStart.ApplicationId = NewUUID(applicationId)
}
if instanceIndex, err := strconv.Atoi(req.Header.Get("X-CF-InstanceIndex")); err == nil {
httpStart.InstanceIndex = proto.Int(instanceIndex)
}
if instanceId := req.Header.Get("X-CF-InstanceID"); instanceId != "" {
httpStart.InstanceId = &instanceId
}
return httpStart
}
func (proxy *Proxy) isAuthorized(appId, authToken string, clientAddress string) (bool, *logmessage.LogMessage) {
newLogMessage := func(message []byte) *logmessage.LogMessage {
currentTime := time.Now()
messageType := logmessage.LogMessage_ERR
return &logmessage.LogMessage{
Message: message,
AppId: proto.String(appId),
MessageType: &messageType,
SourceName: proto.String("LGR"),
Timestamp: proto.Int64(currentTime.UnixNano()),
}
}
if appId == "" {
message := fmt.Sprintf("HttpServer: Did not accept sink connection with invalid app id: %s.", clientAddress)
proxy.logger.Warn(message)
return false, newLogMessage([]byte("Error: Invalid target"))
}
if authToken == "" {
message := fmt.Sprintf("HttpServer: Did not accept sink connection from %s without authorization.", clientAddress)
proxy.logger.Warnf(message)
return false, newLogMessage([]byte("Error: Authorization not provided"))
}
if !proxy.authorize(authToken, appId, proxy.logger) {
message := fmt.Sprintf("HttpServer: Auth token [%s] not authorized to access appId [%s].", authToken, appId)
proxy.logger.Warn(message)
return false, newLogMessage([]byte("Error: Invalid authorization"))
}
return true, nil
}
// Encodes the SnapshotRecoveryRequest to a buffer. Returns the number of bytes
// written and any error that may have occurred.
func (req *SnapshotRecoveryRequest) Encode(w io.Writer) (int, error) {
protoPeers := make([]*protobuf.SnapshotRecoveryRequest_Peer, len(req.Peers))
for i, peer := range req.Peers {
protoPeers[i] = &protobuf.SnapshotRecoveryRequest_Peer{
Name: proto.String(peer.Name),
ConnectionString: proto.String(peer.ConnectionString),
}
}
pb := &protobuf.SnapshotRecoveryRequest{
LeaderName: proto.String(req.LeaderName),
LastIndex: proto.Uint64(req.LastIndex),
LastTerm: proto.Uint64(req.LastTerm),
Peers: protoPeers,
State: req.State,
}
p, err := proto.Marshal(pb)
if err != nil {
return -1, err
}
return w.Write(p)
}
func NewNoaaLogMessage(msgText, appGuid, sourceName string, timestamp time.Time) *events.LogMessage {
messageType := events.LogMessage_ERR
return &events.LogMessage{
Message: []byte(msgText),
AppId: proto.String(appGuid),
MessageType: &messageType,
SourceType: proto.String(sourceName),
Timestamp: proto.Int64(timestamp.UnixNano()),
}
}
func NewLogMessageWithSourceName(t *testing.T, messageString, sourceName, appId string) *LogMessage {
currentTime := time.Now()
messageType := LogMessage_OUT
protoMessage := &LogMessage{
Message: []byte(messageString),
AppId: proto.String(appId),
MessageType: &messageType,
Timestamp: proto.Int64(currentTime.UnixNano()),
SourceName: proto.String(sourceName),
}
return protoMessage
}
func generateLogMessage(messageString, appId string, messageType logmessage.LogMessage_MessageType, sourceName, sourceId string) *logmessage.LogMessage {
currentTime := time.Now()
logMessage := &logmessage.LogMessage{
Message: []byte(messageString),
AppId: proto.String(appId),
MessageType: &messageType,
SourceName: proto.String(sourceName),
SourceId: proto.String(sourceId),
Timestamp: proto.Int64(currentTime.UnixNano()),
}
return logMessage
}
func generateLogMessage(messageString string, appId string) []byte {
messageType := logmessage.LogMessage_ERR
currentTime := time.Now()
logMessage := &logmessage.LogMessage{
Message: []byte(messageString),
AppId: proto.String(appId),
MessageType: &messageType,
SourceName: proto.String("LGR"),
Timestamp: proto.Int64(currentTime.UnixNano()),
}
msg, _ := proto.Marshal(logMessage)
return msg
}
func startAppWithInstancesAndErrors(t *testing.T, app cf.Application, instances [][]cf.ApplicationInstance, errorCodes []string) (ui *testterm.FakeUI, appRepo *testapi.FakeApplicationRepository, reqFactory *testreq.FakeReqFactory) {
token, err := testconfig.CreateAccessTokenWithTokenInfo(configuration.TokenInfo{
Username: "******",
})
assert.NoError(t, err)
config := &configuration.Configuration{
Space: cf.Space{Name: "my-space"},
Organization: cf.Organization{Name: "my-org"},
AccessToken: token,
ApplicationStartTimeout: 2,
}
appRepo = &testapi.FakeApplicationRepository{
FindByNameApp: app,
GetInstancesResponses: instances,
GetInstancesErrorCodes: errorCodes,
}
currentTime := time.Now()
messageType := logmessage.LogMessage_ERR
sourceType := logmessage.LogMessage_DEA
logMessage1 := logmessage.LogMessage{
Message: []byte("Log Line 1"),
AppId: proto.String(app.Guid),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
logMessage2 := logmessage.LogMessage{
Message: []byte("Log Line 2"),
AppId: proto.String(app.Guid),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
logRepo := &testapi.FakeLogsRepository{
TailLogMessages: []logmessage.LogMessage{
logMessage1,
logMessage2,
},
}
args := []string{"my-app"}
reqFactory = &testreq.FakeReqFactory{Application: app}
ui = callStart(args, config, reqFactory, appRepo, logRepo)
return
}
func (c *Connection) Run(handle, script string) (*warden.RunResponse, error) {
res, err := c.RoundTrip(
&warden.RunRequest{
Handle: proto.String(handle),
Script: proto.String(script),
},
&warden.RunResponse{},
)
if err != nil {
return nil, err
}
return res.(*warden.RunResponse), nil
}
func NewLogMessage(msgText, appGuid, sourceName string, timestamp time.Time) (msg *logmessage.Message) {
messageType := logmessage.LogMessage_ERR
logMsg := logmessage.LogMessage{
Message: []byte(msgText),
AppId: proto.String(appGuid),
MessageType: &messageType,
SourceName: proto.String(sourceName),
Timestamp: proto.Int64(timestamp.UnixNano()),
}
data, _ := proto.Marshal(&logMsg)
msg, _ = logmessage.ParseMessage(data)
return
}
func Wrap(e events.Event, origin string) (*events.Envelope, error) {
if origin == "" {
return nil, errors.New("Event not emitted due to missing origin information")
}
envelope := &events.Envelope{Origin: proto.String(origin)}
switch e.(type) {
case *events.Heartbeat:
envelope.EventType = events.Envelope_Heartbeat.Enum()
envelope.Heartbeat = e.(*events.Heartbeat)
case *events.HttpStart:
envelope.EventType = events.Envelope_HttpStart.Enum()
envelope.HttpStart = e.(*events.HttpStart)
case *events.HttpStop:
envelope.EventType = events.Envelope_HttpStop.Enum()
envelope.HttpStop = e.(*events.HttpStop)
case *events.ValueMetric:
envelope.EventType = events.Envelope_ValueMetric.Enum()
envelope.ValueMetric = e.(*events.ValueMetric)
case *events.CounterEvent:
envelope.EventType = events.Envelope_CounterEvent.Enum()
envelope.CounterEvent = e.(*events.CounterEvent)
default:
return nil, errors.New("Cannot create envelope for unknown event type")
}
return envelope, nil
}
func Wrap(e events.Event, origin string) (*events.Envelope, error) {
if origin == "" {
return nil, ErrorMissingOrigin
}
envelope := &events.Envelope{Origin: proto.String(origin)}
switch e := e.(type) {
case *events.Heartbeat:
envelope.EventType = events.Envelope_Heartbeat.Enum()
envelope.Heartbeat = e
case *events.HttpStart:
envelope.EventType = events.Envelope_HttpStart.Enum()
envelope.HttpStart = e
case *events.HttpStop:
envelope.EventType = events.Envelope_HttpStop.Enum()
envelope.HttpStop = e
case *events.ValueMetric:
envelope.EventType = events.Envelope_ValueMetric.Enum()
envelope.ValueMetric = e
case *events.CounterEvent:
envelope.EventType = events.Envelope_CounterEvent.Enum()
envelope.CounterEvent = e
case *events.LogMessage:
envelope.EventType = events.Envelope_LogMessage.Enum()
envelope.LogMessage = e
default:
return nil, ErrorUnknownEventType
}
return envelope, nil
}
func (s *WardenServer) streamProcessToConnection(processID uint32, stream <-chan warden.ProcessStream, conn net.Conn) proto.Message {
for payload := range stream {
if payload.ExitStatus != nil {
return &protocol.ProcessPayload{
ProcessId: proto.Uint32(processID),
ExitStatus: proto.Uint32(*payload.ExitStatus),
}
}
var payloadSource protocol.ProcessPayload_Source
switch payload.Source {
case warden.ProcessStreamSourceStdout:
payloadSource = protocol.ProcessPayload_stdout
case warden.ProcessStreamSourceStderr:
payloadSource = protocol.ProcessPayload_stderr
case warden.ProcessStreamSourceStdin:
payloadSource = protocol.ProcessPayload_stdin
}
protocol.Messages(&protocol.ProcessPayload{
ProcessId: proto.Uint32(processID),
Source: &payloadSource,
Data: proto.String(string(payload.Data)),
}).WriteTo(conn)
}
return nil
}
func (c *Connection) Spawn(handle, script string, discardOutput bool) (*warden.SpawnResponse, error) {
res, err := c.RoundTrip(
&warden.SpawnRequest{
Handle: proto.String(handle),
Script: proto.String(script),
DiscardOutput: proto.Bool(discardOutput),
},
&warden.SpawnResponse{},
)
if err != nil {
return nil, err
}
return res.(*warden.SpawnResponse), nil
}
// Creates a new log entry associated with a log.
func newLogEntry(log *Log, event *ev, index uint64, term uint64, command Command) (*LogEntry, error) {
var buf bytes.Buffer
var commandName string
if command != nil {
commandName = command.CommandName()
if encoder, ok := command.(CommandEncoder); ok {
if err := encoder.Encode(&buf); err != nil {
return nil, err
}
} else {
json.NewEncoder(&buf).Encode(command)
}
}
pb := &protobuf.LogEntry{
Index: proto.Uint64(index),
Term: proto.Uint64(term),
CommandName: proto.String(commandName),
Command: buf.Bytes(),
}
e := &LogEntry{
pb: pb,
log: log,
event: event,
}
return e, nil
}
func TestLogsTailsTheAppLogs(t *testing.T) {
app := cf.Application{Name: "my-app", Guid: "my-app-guid"}
currentTime := time.Now()
messageType := logmessage.LogMessage_ERR
deaSourceType := logmessage.LogMessage_DEA
deaSourceId := "42"
deaLogMessage := logmessage.LogMessage{
Message: []byte("Log Line 1"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &deaSourceType,
SourceId: &deaSourceId,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
logs := []logmessage.LogMessage{deaLogMessage}
reqFactory, logsRepo := getLogsDependencies()
reqFactory.Application = app
logsRepo.TailLogMessages = logs
ui := callLogs(t, []string{"my-app"}, reqFactory, logsRepo)
assert.Equal(t, reqFactory.ApplicationName, "my-app")
assert.Equal(t, app, logsRepo.AppLogged)
assert.Equal(t, len(ui.Outputs), 2)
assert.Contains(t, ui.Outputs[0], "Connected, tailing logs for app")
assert.Contains(t, ui.Outputs[0], "my-app")
assert.Contains(t, ui.Outputs[0], "my-org")
assert.Contains(t, ui.Outputs[0], "my-space")
assert.Contains(t, ui.Outputs[0], "my-user")
assert.Contains(t, ui.Outputs[1], "Log Line 1")
}
func (c *Connection) CopyIn(handle, src, dst string) (*warden.CopyInResponse, error) {
res, err := c.RoundTrip(
&warden.CopyInRequest{
Handle: proto.String(handle),
SrcPath: proto.String(src),
DstPath: proto.String(dst),
},
&warden.CopyInResponse{},
)
if err != nil {
return nil, err
}
return res.(*warden.CopyInResponse), nil
}
func TestStringEscaping(t *testing.T) {
testCases := []struct {
in *pb.Strings
out string
}{
{
// Test data from C++ test (TextFormatTest.StringEscape).
// Single divergence: we don't escape apostrophes.
&pb.Strings{StringField: proto.String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces")},
"string_field: \"\\\"A string with ' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"\n",
},
{
// Test data from the same C++ test.
&pb.Strings{StringField: proto.String("\350\260\267\346\255\214")},
"string_field: \"\\350\\260\\267\\346\\255\\214\"\n",
},
{
// Some UTF-8.
&pb.Strings{StringField: proto.String("\x00\x01\xff\x81")},
`string_field: "\000\001\377\201"` + "\n",
},
}
for i, tc := range testCases {
var buf bytes.Buffer
if err := proto.MarshalText(&buf, tc.in); err != nil {
t.Errorf("proto.MarsalText: %v", err)
continue
}
s := buf.String()
if s != tc.out {
t.Errorf("#%d: Got:\n%s\nExpected:\n%s\n", i, s, tc.out)
continue
}
// Check round-trip.
pb := new(pb.Strings)
if err := proto.UnmarshalText(s, pb); err != nil {
t.Errorf("#%d: UnmarshalText: %v", i, err)
continue
}
if !proto.Equal(pb, tc.in) {
t.Errorf("#%d: Round-trip failed:\nstart: %v\n end: %v", i, tc.in, pb)
}
}
}
func init() {
ext := &pb.Ext{
Data: proto.String("extension"),
}
if err := proto.SetExtension(cloneTestMessage, pb.E_Ext_More, ext); err != nil {
panic("SetExtension: " + err.Error())
}
}
func TestThatSignatureDoesNotValidateWhenItDoesntMatch(t *testing.T) {
envelope := &LogEnvelope{
LogMessage: &LogMessage{},
RoutingKey: proto.String("app_id"),
Signature: []byte{0, 1, 2}, //some bad signature
}
assert.False(t, envelope.VerifySignature("super-secret"))
}
func MarshalledLogEnvelope(unmarshalledMessage *logmessage.LogMessage, secret string) []byte {
envelope := &logmessage.LogEnvelope{
LogMessage: unmarshalledMessage,
RoutingKey: proto.String(*unmarshalledMessage.AppId),
}
envelope.SignEnvelope(secret)
return marshalProtoBuf(envelope)
}
func TestLogsOutputsRecentLogs(t *testing.T) {
app := cf.Application{Name: "my-app", Guid: "my-app-guid"}
currentTime := time.Now()
messageType := logmessage.LogMessage_ERR
sourceType := logmessage.LogMessage_DEA
logMessage1 := logmessage.LogMessage{
Message: []byte("Log Line 1"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
logMessage2 := logmessage.LogMessage{
Message: []byte("Log Line 2"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
recentLogs := []logmessage.LogMessage{
logMessage1,
logMessage2,
}
reqFactory, logsRepo := getLogsDependencies()
reqFactory.Application = app
logsRepo.RecentLogs = recentLogs
ui := callLogs(t, []string{"--recent", "my-app"}, reqFactory, logsRepo)
assert.Equal(t, reqFactory.ApplicationName, "my-app")
assert.Equal(t, app, logsRepo.AppLogged)
assert.Equal(t, len(ui.Outputs), 3)
assert.Contains(t, ui.Outputs[0], "Connected, dumping recent logs for app")
assert.Contains(t, ui.Outputs[0], "my-app")
assert.Contains(t, ui.Outputs[0], "my-org")
assert.Contains(t, ui.Outputs[0], "my-space")
assert.Contains(t, ui.Outputs[0], "my-user")
assert.Contains(t, ui.Outputs[1], "Log Line 1")
assert.Contains(t, ui.Outputs[2], "Log Line 2")
}
func (c *Connection) Info(handle string) (*warden.InfoResponse, error) {
res, err := c.RoundTrip(&warden.InfoRequest{
Handle: proto.String(handle),
}, &warden.InfoResponse{})
if err != nil {
return nil, err
}
return res.(*warden.InfoResponse), nil
}
func (e *LoggregatorEmitter) newLogEnvelope(appId string, message *events.LogMessage) (*logmessage.LogEnvelope, error) {
envelope := &logmessage.LogEnvelope{
LogMessage: convertToOldFormat(message),
RoutingKey: proto.String(appId),
Signature: []byte{},
}
err := envelope.SignEnvelope(e.sharedSecret)
return envelope, err
}
func makeLogMessage(appId, message, sourceType, sourceInstance string, messageType events.LogMessage_MessageType) *events.LogMessage {
return &events.LogMessage{
Message: []byte(message),
AppId: proto.String(appId),
MessageType: &messageType,
SourceType: &sourceType,
SourceInstance: &sourceInstance,
Timestamp: proto.Int64(time.Now().UnixNano()),
}
}
func (rs *RuntimeStats) emit(name string, value float64) {
err := rs.eventEmitter.Emit(&events.ValueMetric{
Name: &name,
Value: &value,
Unit: proto.String("count"),
})
if err != nil {
log.Printf("RuntimeStats: failed to emit: %v", err)
}
}
func generateGoGoProtobufMessages() []proto.Message {
return []proto.Message{
&example.GoGoProtobufTestMessage1{
F0: proto.Int32(1),
F1: proto.String("hello"),
F2: proto.Float32(4.2)},
&example.GoGoProtobufTestMessage2{
F0: proto.Int32(2),
F1: proto.String("world"),
F2: proto.Float32(2.4)},
&example.GoGoProtobufTestMessage3{
F0: proto.Int32(3),
F1: proto.String("test"),
F2: proto.String("4.2")},
&example.GoGoProtobufTestMessage4{
F0: proto.Int32(3),
F1: proto.String("codec")},
}
}
func (ls *loggingStream) listen() {
newLogMessage := func(message []byte) *logmessage.LogMessage {
currentTime := time.Now()
sourceType := logmessage.LogMessage_WARDEN_CONTAINER
return &logmessage.LogMessage{
Message: message,
AppId: proto.String(ls.inst.applicationId),
DrainUrls: ls.inst.drainUrls,
MessageType: &ls.messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
}
socket := func(messageType logmessage.LogMessage_MessageType) (net.Conn, error) {
return net.Dial("unix", filepath.Join(ls.inst.identifier(), socketName(messageType)))
}
go func() {
connection, err := socket(ls.messageType)
if err != nil {
ls.logger.Errorf("Error while dialing into socket %s, %s", ls.messageType, err)
return
}
defer func() {
connection.Close()
ls.logger.Infof("Stopped reading from socket %s", ls.messageType)
}()
buffer := make([]byte, bufferSize)
for {
readCount, err := connection.Read(buffer)
if readCount == 0 && err != nil {
ls.logger.Infof("Error while reading from socket %s, %s", ls.messageType, err)
break
}
ls.logger.Debugf("Read %d bytes from instance socket %s", readCount, ls.messageType)
atomic.AddUint64(ls.messagesReceived, 1)
atomic.AddUint64(ls.bytesReceived, uint64(readCount))
ls.loggregatorClient.IncLogStreamRawByteCount(uint64(readCount))
data, err := proto.Marshal(newLogMessage(buffer[0:readCount]))
if err != nil {
ls.logger.Errorf("Error marshalling log message: %s", err)
}
ls.loggregatorClient.IncLogStreamPbByteCount(uint64(len(data)))
ls.loggregatorClient.Send(data)
ls.logger.Debugf("Sent %d bytes to loggregator client", readCount)
runtime.Gosched()
}
}()
}
func TestLogsTailsTheAppLogs(t *testing.T) {
app := cf.Application{Name: "my-app", Guid: "my-app-guid"}
///////////////
currentTime := time.Now()
messageType := logmessage.LogMessage_ERR
sourceType := logmessage.LogMessage_DEA
logMessage1 := &logmessage.LogMessage{
Message: []byte("Log Line 1"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
logMessage2 := &logmessage.LogMessage{
Message: []byte("Log Line 2"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
/////////////////
logs := []*logmessage.LogMessage{
logMessage1,
logMessage2,
}
reqFactory, logsRepo := getDefaultLogsDependencies()
reqFactory.Application = app
logsRepo.TailLogMessages = logs
ui := callLogs([]string{"my-app"}, reqFactory, logsRepo)
assert.Equal(t, reqFactory.ApplicationName, "my-app")
assert.Equal(t, app, logsRepo.AppLogged)
assert.Equal(t, len(ui.Outputs), 3)
assert.Contains(t, ui.Outputs[0], "Connected")
assert.Contains(t, ui.Outputs[1], "Log Line 1")
assert.Contains(t, ui.Outputs[2], "Log Line 2")
}
func TestRecentLogsWithAppName(t *testing.T) {
config := &configuration.Configuration{}
app := cf.Application{Name: "my-app", Guid: "my-app-guid"}
appRepo := &testhelpers.FakeApplicationRepository{AppByName: app}
///////////////
currentTime := time.Now()
messageType := logmessage.LogMessage_ERR
sourceType := logmessage.LogMessage_DEA
logMessage1 := &logmessage.LogMessage{
Message: []byte("Log Line 1"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
logMessage2 := &logmessage.LogMessage{
Message: []byte("Log Line 2"),
AppId: proto.String("my-app"),
MessageType: &messageType,
SourceType: &sourceType,
Timestamp: proto.Int64(currentTime.UnixNano()),
}
/////////////////
recentLogs := []*logmessage.LogMessage{
logMessage1,
logMessage2,
}
logsRepo := &testhelpers.FakeLogsRepository{RecentLogs: recentLogs}
reqFactory := &testhelpers.FakeReqFactory{Application: app}
ui := callRecentLogs([]string{"my-app"}, config, reqFactory, appRepo, logsRepo)
assert.Equal(t, reqFactory.ApplicationName, "my-app")
assert.Equal(t, app, logsRepo.AppLogged)
assert.Equal(t, len(ui.Outputs), 2)
assert.Contains(t, ui.Outputs[0], "Log Line 1")
assert.Contains(t, ui.Outputs[1], "Log Line 2")
}