// ElasticsearchMessageEncoder defines the encoding from SubscribeResponse to
// sarama.ProducerMessage for Elasticsearch
func ElasticsearchMessageEncoder(topic string, key sarama.Encoder, dataset string,
message proto.Message) (*sarama.ProducerMessage, error) {
response, ok := message.(*pb.SubscribeResponse)
if !ok {
return nil, UnhandledMessageError{message: message}
}
update := response.GetUpdate()
if update == nil {
return nil, UnhandledSubscribeResponseError{response: response}
}
updateMap, err := openconfig.NotificationToMap(dataset, update,
elasticsearch.EscapeFieldName)
if err != nil {
return nil, err
}
// Convert time to ms to make Elasticsearch happy
updateMap["timestamp"] = updateMap["timestamp"].(int64) / 1000000
updateJSON, err := json.Marshal(updateMap)
if err != nil {
return nil, err
}
glog.V(9).Infof("kafka: %s", updateJSON)
return &sarama.ProducerMessage{
Topic: topic,
Key: key,
Value: sarama.ByteEncoder(updateJSON),
Metadata: kafka.Metadata{StartTime: time.Unix(0, update.Timestamp), NumMessages: 1},
}, nil
}
func (c *client) Run(ch chan<- *pb.LanzRecord) {
for !c.stopping {
if err := c.conn.Connect(); err != nil && !c.stopping {
glog.V(1).Infof("Can't connect to LANZ server: %v", err)
time.Sleep(c.backoff)
continue
}
glog.V(1).Infof("Connected successfully to LANZ server: %v", c.addr)
if err := c.read(bufio.NewReader(c.conn), ch); err != nil && !c.stopping {
if err != io.EOF && err != io.ErrUnexpectedEOF {
glog.Errorf("Error receiving LANZ events: %v", err)
}
c.conn.Close()
time.Sleep(c.backoff)
}
}
close(ch)
}
func (p *producer) produceNotification(protoMessage proto.Message) error {
message, err := p.encoder(p.topic, p.key, p.dataset, protoMessage)
if err != nil {
return err
}
select {
case p.kafkaProducer.Input() <- message:
glog.V(9).Infof("Message produced to Kafka: %s", message)
return nil
case <-p.done:
return nil
}
}
func main() {
username, password, subscriptions, addrs, opts := client.ParseFlags()
if *getFlag != "" {
c := client.New(username, password, addrs[0], opts)
for _, notification := range c.Get(*getFlag) {
var notifStr string
if *jsonFlag {
var err error
if notifStr, err = openconfig.NotificationToJSON(notification); err != nil {
glog.Fatal(err)
}
} else {
notifStr = notification.String()
}
fmt.Println(notifStr)
}
return
}
publish := func(addr string, message proto.Message) {
resp, ok := message.(*pb.SubscribeResponse)
if !ok {
glog.Errorf("Unexpected type of message: %T", message)
return
}
if resp.GetHeartbeat() != nil && !glog.V(1) {
return // Log heartbeats with verbose logging only.
}
var respTxt string
var err error
if *jsonFlag {
respTxt, err = openconfig.SubscribeResponseToJSON(resp)
if err != nil {
glog.Fatal(err)
}
} else {
respTxt = proto.MarshalTextString(resp)
}
fmt.Println(respTxt)
}
wg := new(sync.WaitGroup)
for _, addr := range addrs {
wg.Add(1)
c := client.New(username, password, addr, opts)
go c.Subscribe(wg, subscriptions, publish)
}
wg.Wait()
}
func pushToOpenTSDB(addr string, conn OpenTSDBConn, config *Config,
notif *openconfig.Notification) {
if notif.Timestamp <= 0 {
glog.Fatalf("Invalid timestamp %d in %s", notif.Timestamp, notif)
}
host := addr[:strings.IndexRune(addr, ':')]
prefix := "/" + strings.Join(notif.Prefix.Element, "/")
for _, update := range notif.Update {
if update.Value == nil || update.Value.Type != openconfig.Type_JSON {
glog.V(9).Infof("Ignoring incompatible update value in %s", update)
continue
}
value := parseValue(update)
if value == nil {
glog.V(9).Infof("Ignoring non-numeric value in %s", update)
continue
}
path := prefix + "/" + strings.Join(update.Path.Element, "/")
metricName, tags := config.Match(path)
if metricName == "" {
glog.V(8).Infof("Ignoring unmatched update at %s: %+v", path, update.Value)
continue
}
tags["host"] = host
conn.Put(&DataPoint{
Metric: metricName,
Timestamp: uint64(notif.Timestamp),
Value: value,
Tags: tags,
})
}
}
// Subscribe sends subscriptions, and consumes responses.
// The given publish function is used to publish SubscribeResponses received
// for the given subscriptions, when connected to the given host, with the
// given user/pass pair, or the client-side cert specified in the gRPC opts.
// This function does not normally return so it should probably be run in its
// own goroutine. When this function returns, the given WaitGroup is marked
// as done.
func (c *Client) Subscribe(wg *sync.WaitGroup, subscriptions []string,
publish PublishFunc) {
defer wg.Done()
stream, err := c.client.Subscribe(c.ctx)
if err != nil {
glog.Fatalf("Subscribe failed: %s", err)
}
defer stream.CloseSend()
for _, path := range subscriptions {
sub := &openconfig.SubscribeRequest{
Request: &openconfig.SubscribeRequest_Subscribe{
Subscribe: &openconfig.SubscriptionList{
Subscription: []*openconfig.Subscription{
&openconfig.Subscription{
Path: &openconfig.Path{Element: strings.Split(path, "/")},
},
},
},
},
}
glog.Infof("Sending subscribe request: %s", sub)
err = stream.Send(sub)
if err != nil {
glog.Fatalf("Failed to subscribe: %s", err)
}
}
for {
resp, err := stream.Recv()
if err != nil {
if err != io.EOF {
glog.Fatalf("Error received from the server: %s", err)
}
return
}
switch resp := resp.Response.(type) {
case *openconfig.SubscribeResponse_SyncResponse:
if !resp.SyncResponse {
panic("initial sync failed," +
" check that you're using a client compatible with the server")
}
}
glog.V(3).Info(resp)
publish(c.device, resp)
}
}