func Announce(localIP string, ttl time.Duration, config *config.Config, storeAdapter storeadapter.StoreAdapter, logger *gosteno.Logger) chan (chan bool) {
dopplerMetaBytes, err := buildDopplerMeta(localIP, config)
if err != nil {
panic(err)
}
key := fmt.Sprintf("%s/%s/%s/%d", META_ROOT, config.Zone, config.JobName, config.Index)
logger.Debugf("Starting Health Status Updates to Store: %s", key)
node := storeadapter.StoreNode{
Key: key,
Value: dopplerMetaBytes,
TTL: uint64(ttl.Seconds()),
}
// Call to create to make sure node is created before we return
storeAdapter.Create(node)
status, stopChan, err := storeAdapter.MaintainNode(node)
if err != nil {
panic(err)
}
// The status channel needs to be drained to maintain the node within the etcd cluster
go func() {
for stat := range status {
logger.Debugf("Health updates channel pushed %v at time %v", stat, time.Now())
}
}()
return stopChan
}
func NewLoggregatorClient(loggregatorAddress string, logger *gosteno.Logger, bufferSize int) LoggregatorClient {
loggregatorClient := &udpLoggregatorClient{}
la, err := net.ResolveUDPAddr("udp", loggregatorAddress)
if err != nil {
logger.Fatalf("Error resolving loggregator address %s, %s", loggregatorAddress, err)
}
connection, err := net.ListenPacket("udp", "")
if err != nil {
logger.Fatalf("Error opening udp stuff")
}
loggregatorClient.loggregatorAddress = la.IP.String()
loggregatorClient.sendChannel = make(chan []byte, bufferSize)
go func() {
for dataToSend := range loggregatorClient.sendChannel {
if len(dataToSend) > 0 {
writeCount, err := connection.WriteTo(dataToSend, la)
if err != nil {
logger.Errorf("Writing to loggregator %s failed %s", loggregatorAddress, err)
continue
}
logger.Debugf("Wrote %d bytes to %s", writeCount, loggregatorAddress)
atomic.AddUint64(&loggregatorClient.sentMessageCount, 1)
atomic.AddUint64(&loggregatorClient.sentByteCount, uint64(writeCount))
} else {
logger.Debugf("Skipped writing of 0 byte message to %s", loggregatorAddress)
}
}
}()
return loggregatorClient
}
func NewLoggregatorClient(loggregatorAddress string, logger *gosteno.Logger, bufferSize int) *udpLoggregatorClient {
loggregatorClient := &udpLoggregatorClient{make(chan []byte, bufferSize)}
connection, err := net.Dial("udp", loggregatorAddress)
if err != nil {
logger.Fatalf("Error resolving loggregator address %s, %s", loggregatorAddress, err)
panic(err)
}
go func() {
for {
dataToSend := <-loggregatorClient.sendChannel
if len(dataToSend) > 0 {
writeCount, err := connection.Write(dataToSend)
logger.Debugf("Wrote %d bytes to %s", writeCount, loggregatorAddress)
if err != nil {
logger.Errorf("Writing to loggregator %s failed %s", loggregatorAddress, err)
}
} else {
logger.Debugf("Skipped writing of 0 byte message to %s", loggregatorAddress)
}
}
}()
return loggregatorClient
}
func StartHeartbeats(localIp string, ttl time.Duration, config *config.Config, storeAdapter storeadapter.StoreAdapter, logger *gosteno.Logger) (stopChan chan (chan bool)) {
if len(config.EtcdUrls) == 0 {
return
}
if storeAdapter == nil {
panic("store adapter is nil")
}
logger.Debugf("Starting Health Status Updates to Store: /healthstatus/doppler/%s/%s/%d", config.Zone, config.JobName, config.Index)
status, stopChan, err := storeAdapter.MaintainNode(storeadapter.StoreNode{
Key: fmt.Sprintf("/healthstatus/doppler/%s/%s/%d", config.Zone, config.JobName, config.Index),
Value: []byte(localIp),
TTL: uint64(ttl.Seconds()),
})
if err != nil {
panic(err)
}
go func() {
for stat := range status {
logger.Debugf("Health updates channel pushed %v at time %v", stat, time.Now())
}
}()
return stopChan
}
func sendMessagesToWebsocket(logMessages []*logmessage.Message, ws *websocket.Conn, logger *gosteno.Logger) {
for _, message := range logMessages {
err := ws.WriteMessage(websocket.BinaryMessage, message.GetRawMessage())
if err != nil {
logger.Debugf("Dump Sink %s: Error when trying to send data to sink %s. Requesting close. Err: %v", ws.RemoteAddr(), err)
} else {
logger.Debugf("Dump Sink %s: Successfully sent data", ws.RemoteAddr())
}
}
}
func makeOutgoingProxy(ipAddress string, config *Config, logger *gosteno.Logger) *trafficcontroller.Proxy {
authorizer := authorization.NewLogAccessAuthorizer(config.ApiHost, config.SkipCertVerify)
logger.Debugf("Output Proxy Startup: Number of zones: %v", len(config.Loggregators))
hashers := makeHashers(config.Loggregators, config.OutgoingPort, logger)
logger.Debugf("Output Proxy Startup: Number of hashers for the proxy: %v", len(hashers))
proxy := trafficcontroller.NewProxy(net.JoinHostPort(ipAddress, strconv.FormatUint(uint64(config.OutgoingPort), 10)), hashers, authorizer, logger)
return proxy
}
func dumpMessagesFromChannelToWebsocket(dumpChan <-chan *logmessage.Message, ws *websocket.Conn, clientAddress net.Addr, logger *gosteno.Logger) {
for message := range dumpChan {
err := websocket.Message.Send(ws, message.GetRawMessage())
if err != nil {
logger.Debugf("Dump Sink %s: Error when trying to send data to sink %s. Requesting close. Err: %v", clientAddress, err)
} else {
logger.Debugf("Dump Sink %s: Successfully sent data", clientAddress)
}
}
}
func makeOutgoingProxy(config *Config, logger *gosteno.Logger) *trafficcontroller.Proxy {
authorizer := authorization.NewLogAccessAuthorizer(config.ApiHost, config.SkipCertVerify)
logger.Debugf("Output Proxy Startup: Number of zones: %v", len(config.Loggregators))
hashers := makeHashers(config.Loggregators, config.LoggregatorOutgoingPort, logger)
logger.Debugf("Output Proxy Startup: Number of hashers for the proxy: %v", len(hashers))
proxy := trafficcontroller.NewProxy(hashers, authorizer, logger)
return proxy
}
func NewSyslogSink(appId string, drainUrl string, givenLogger *gosteno.Logger, syslogWriter SyslogWriter) Sink {
givenLogger.Debugf("Syslog Sink %s: Created for appId [%s]", drainUrl, appId)
return &SyslogSink{
appId: appId,
drainUrl: drainUrl,
logger: givenLogger,
sentMessageCount: new(uint64),
sentByteCount: new(uint64),
listenerChannel: make(chan *logmessage.Message),
syslogWriter: syslogWriter,
}
}
func NewSyslogSink(appId string, drainUrl string, givenLogger *gosteno.Logger, syslogWriter syslogwriter.SyslogWriter, errorChannel chan<- *logmessage.Message) sinks.Sink {
givenLogger.Debugf("Syslog Sink %s: Created for appId [%s]", drainUrl, appId)
return &SyslogSink{
appId: appId,
drainUrl: drainUrl,
logger: givenLogger,
sentMessageCount: new(uint64),
sentByteCount: new(uint64),
syslogWriter: syslogWriter,
errorChannel: errorChannel,
disconnectChannel: make(chan struct{}),
}
}
func NewSyslogSink(appId string, drainUrl string, givenLogger *gosteno.Logger, syslogWriter syslogwriter.Writer, errorHandler func(string, string, string), dropsondeOrigin string, metricUpdateChannel chan<- int64) *SyslogSink {
givenLogger.Debugf("Syslog Sink %s: Created for appId [%s]", drainUrl, appId)
return &SyslogSink{
appId: appId,
drainUrl: drainUrl,
Logger: givenLogger,
syslogWriter: syslogWriter,
handleSendError: errorHandler,
disconnectChannel: make(chan struct{}),
dropsondeOrigin: dropsondeOrigin,
metricUpdateChannel: metricUpdateChannel,
}
}
// NewDropsondeUnmarshallerCollection instantiates a DropsondeUnmarshallerCollection,
// creates the specified number of DropsondeUnmarshaller instances and logs to the
// provided logger.
func NewDropsondeUnmarshallerCollection(logger *gosteno.Logger, size int) *DropsondeUnmarshallerCollection {
var unmarshallers []*DropsondeUnmarshaller
for i := 0; i < size; i++ {
unmarshallers = append(unmarshallers, NewDropsondeUnmarshaller(logger))
}
logger.Debugf("dropsondeUnmarshallerCollection: created %v unmarshallers", size)
return &DropsondeUnmarshallerCollection{
logger: logger,
unmarshallers: unmarshallers,
}
}
func NewSyslogSink(appId string, drainUrl string, givenLogger *gosteno.Logger, messageDrainBufferSize uint, syslogWriter syslogwriter.Writer, errorHandler func(string, string, string), dropsondeOrigin string) *SyslogSink {
givenLogger.Debugf("Syslog Sink %s: Created for appId [%s]", drainUrl, appId)
return &SyslogSink{
appId: appId,
drainUrl: drainUrl,
logger: givenLogger,
messageDrainBufferSize: messageDrainBufferSize,
syslogWriter: syslogWriter,
handleSendError: errorHandler,
disconnectChannel: make(chan struct{}),
dropsondeOrigin: dropsondeOrigin,
}
}
func dial(taskIdentifier string, messageType events.LogMessage_MessageType, logger *gosteno.Logger) (net.Conn, error) {
var err error
var connection net.Conn
for i := 0; i < 10; i++ {
connection, err = net.Dial("unix", filepath.Join(taskIdentifier, socketName(messageType)))
if err == nil {
logger.Debugf("Opened socket %s, %s", messageType, taskIdentifier)
break
}
logger.Debugf("Could not read from socket %s, %s, retrying: %s", messageType, taskIdentifier, err)
time.Sleep(100 * time.Millisecond)
}
return connection, err
}
func sendMessagesToWebsocket(envelopes []*events.Envelope, websocketConnection *gorilla.Conn, logger *gosteno.Logger) {
for _, messageEnvelope := range envelopes {
envelopeBytes, err := proto.Marshal(messageEnvelope)
if err != nil {
logger.Errorf("Websocket Server %s: Error marshalling %s envelope from origin %s: %s", websocketConnection.RemoteAddr(), messageEnvelope.GetEventType().String(), messageEnvelope.GetOrigin(), err.Error())
continue
}
err = websocketConnection.WriteMessage(gorilla.BinaryMessage, envelopeBytes)
if err != nil {
logger.Errorf("Websocket Server %s: Error when trying to send data to sink %s. Err: %v", websocketConnection.RemoteAddr(), err)
} else {
logger.Debugf("Websocket Server %s: Successfully sent data", websocketConnection.RemoteAddr())
}
}
}
func makeHashers(loggregators map[string][]string, outgoingPort uint32, logger *gosteno.Logger) []*hasher.Hasher {
counter := 0
hashers := make([]*hasher.Hasher, 0, len(loggregators))
for _, servers := range loggregators {
logger.Debugf("Output Proxy Startup: Hashing servers: %v Length: %d", servers, len(servers))
if len(servers) == 0 {
continue
}
for index, server := range servers {
logger.Debugf("Output Proxy Startup: Forwarding messages to client from loggregator server [%v] at %v", index, net.JoinHostPort(server, strconv.FormatUint(uint64(outgoingPort), 10)))
servers[index] = net.JoinHostPort(server, strconv.FormatUint(uint64(outgoingPort), 10))
}
hashers = hashers[:(counter + 1)]
hashers[counter] = hasher.NewHasher(servers)
counter++
}
return hashers
}
func NewSyslogSink(appId string, drainUrl string, givenLogger *gosteno.Logger) (Sink, error) {
dl, err := url.Parse(drainUrl)
if err != nil {
return nil, err
}
sysLogger, err := dial("tcp", dl.Host, appId, givenLogger)
if err != nil {
return nil, err
}
givenLogger.Debugf("Syslog Sink %s: Created for appId [%s]", drainUrl, appId)
return &syslogSink{
appId: appId,
sysLogger: sysLogger,
drainUrl: drainUrl,
logger: givenLogger,
sentMessageCount: new(uint64),
sentByteCount: new(uint64),
listenerChannel: make(chan *logmessage.Message),
}, nil
}
func AnnounceLegacy(localIP string, ttl time.Duration, config *config.Config, storeAdapter storeadapter.StoreAdapter, logger *gosteno.Logger) chan (chan bool) {
key := fmt.Sprintf("%s/%s/%s/%d", LEGACY_ROOT, config.Zone, config.JobName, config.Index)
status, stopChan, err := storeAdapter.MaintainNode(storeadapter.StoreNode{
Key: key,
Value: []byte(localIP),
TTL: uint64(ttl.Seconds()),
})
if err != nil {
panic(err)
}
// The status channel needs to be drained to maintain the node within the etcd cluster
go func() {
for stat := range status {
logger.Debugf("Health updates channel pushed %v at time %v", stat, time.Now())
}
}()
return stopChan
}
func sendMessagesToWebsocket(endpoint string, envelopes []*events.Envelope, websocketConnection *gorilla.Conn, batcher Batcher, logger *gosteno.Logger) {
for _, messageEnvelope := range envelopes {
envelopeBytes, err := proto.Marshal(messageEnvelope)
if err != nil {
logger.Errorf("Websocket Server %s: Error marshalling %s envelope from origin %s: %s", websocketConnection.RemoteAddr(), messageEnvelope.GetEventType().String(), messageEnvelope.GetOrigin(), err.Error())
continue
}
err = websocketConnection.WriteMessage(gorilla.BinaryMessage, envelopeBytes)
if err != nil {
logger.Errorf("Websocket Server %s: Error when trying to send data to sink %s. Err: %v", websocketConnection.RemoteAddr(), err)
continue
}
batcher.BatchCounter("sentEnvelopes").
SetTag("protocol", "ws").
SetTag("event_type", messageEnvelope.GetEventType().String()).
SetTag("endpoint", endpoint).
Increment()
logger.Debugf("Websocket Server %s: Successfully sent data", websocketConnection.RemoteAddr())
}
}
func NewUDPClient(logger *gosteno.Logger, address string, bufferSize int) (Client, error) {
loggregatorClient := &udpClient{
address: address,
}
la, err := net.ResolveUDPAddr("udp", address)
if err != nil {
return nil, err
}
connection, err := net.ListenPacket("udp", "")
if err != nil {
return nil, err
}
loggregatorClient.sendChannel = make(chan []byte, bufferSize)
loggregatorClient.doneChannel = make(chan struct{})
go func() {
for dataToSend := range loggregatorClient.sendChannel {
if len(dataToSend) == 0 {
logger.Debugf("Skipped writing of 0 byte message to %s", address)
continue
}
writeCount, err := connection.WriteTo(dataToSend, la)
if err != nil {
logger.Errorf("Writing to loggregator %s failed %s", address, err)
continue
}
logger.Debugf("Wrote %d bytes to %s", writeCount, address)
}
close(loggregatorClient.doneChannel)
}()
return loggregatorClient, nil
}
func makeIncomingRouter(config *Config, logger *gosteno.Logger) *trafficcontroller.Router {
serversForZone := config.Loggregators[config.Zone]
servers := make([]string, len(serversForZone))
for index, server := range serversForZone {
logger.Debugf("Incoming Router Startup: Forwarding messages from source to loggregator server [%v] at %v", index, net.JoinHostPort(server, strconv.FormatUint(uint64(config.IncomingPort), 10)))
servers[index] = net.JoinHostPort(serversForZone[index], strconv.FormatUint(uint64(config.IncomingPort), 10))
}
logger.Debugf("Incoming Router Startup: Loggregator Servers in the zone %s: %v", config.Zone, servers)
h := hasher.NewHasher(servers)
logger.Debugf("Incoming Router Startup: Hashed Loggregator Server in the zone: %v", h.LoggregatorServers())
logger.Debugf("Incoming Router Startup: Going to start incoming router on %v", config.Host)
router, err := trafficcontroller.NewRouter(config.Host, h, config.Config, logger)
if err != nil {
panic(err)
}
return router
}
// Debugf is a helper to avoid logging anything if the log level is not debug.
// It should be scrapped if/when we switch logging libraries to a library that
// doesn't do any processing if the log won't be output.
func Debugf(logger *gosteno.Logger, msg string, inputs ...interface{}) {
switch logger.Level() {
case gosteno.LOG_DEBUG, gosteno.LOG_DEBUG1, gosteno.LOG_DEBUG2:
logger.Debugf(msg, inputs...)
}
}