Exemplo n.º 1
0
// ElasticsearchMessageEncoder defines the encoding from SubscribeResponse to
// sarama.ProducerMessage for Elasticsearch
func ElasticsearchMessageEncoder(topic string, key sarama.Encoder, dataset string,
	message proto.Message) (*sarama.ProducerMessage, error) {
	response, ok := message.(*pb.SubscribeResponse)
	if !ok {
		return nil, UnhandledMessageError{message: message}
	}
	update := response.GetUpdate()
	if update == nil {
		return nil, UnhandledSubscribeResponseError{response: response}
	}
	updateMap, err := openconfig.NotificationToMap(dataset, update,
		elasticsearch.EscapeFieldName)
	if err != nil {
		return nil, err
	}
	// Convert time to ms to make Elasticsearch happy
	updateMap["timestamp"] = updateMap["timestamp"].(int64) / 1000000
	updateJSON, err := json.Marshal(updateMap)
	if err != nil {
		return nil, err
	}
	glog.V(9).Infof("kafka: %s", updateJSON)
	return &sarama.ProducerMessage{
		Topic:    topic,
		Key:      key,
		Value:    sarama.ByteEncoder(updateJSON),
		Metadata: kafka.Metadata{StartTime: time.Unix(0, update.Timestamp), NumMessages: 1},
	}, nil
}
Exemplo n.º 2
0
func (c *client) Run(ch chan<- *pb.LanzRecord) {
	for !c.stopping {
		if err := c.conn.Connect(); err != nil && !c.stopping {
			glog.V(1).Infof("Can't connect to LANZ server: %v", err)
			time.Sleep(c.backoff)
			continue
		}
		glog.V(1).Infof("Connected successfully to LANZ server: %v", c.addr)
		if err := c.read(bufio.NewReader(c.conn), ch); err != nil && !c.stopping {
			if err != io.EOF && err != io.ErrUnexpectedEOF {
				glog.Errorf("Error receiving LANZ events: %v", err)
			}
			c.conn.Close()
			time.Sleep(c.backoff)
		}
	}

	close(ch)
}
Exemplo n.º 3
0
func (p *producer) produceNotification(protoMessage proto.Message) error {
	message, err := p.encoder(p.topic, p.key, p.dataset, protoMessage)
	if err != nil {
		return err
	}
	select {
	case p.kafkaProducer.Input() <- message:
		glog.V(9).Infof("Message produced to Kafka: %s", message)
		return nil
	case <-p.done:
		return nil
	}
}
Exemplo n.º 4
0
func main() {
	username, password, subscriptions, addrs, opts := client.ParseFlags()

	if *getFlag != "" {
		c := client.New(username, password, addrs[0], opts)
		for _, notification := range c.Get(*getFlag) {
			var notifStr string
			if *jsonFlag {
				var err error
				if notifStr, err = openconfig.NotificationToJSON(notification); err != nil {
					glog.Fatal(err)
				}
			} else {
				notifStr = notification.String()
			}
			fmt.Println(notifStr)

		}
		return
	}

	publish := func(addr string, message proto.Message) {
		resp, ok := message.(*pb.SubscribeResponse)
		if !ok {
			glog.Errorf("Unexpected type of message: %T", message)
			return
		}
		if resp.GetHeartbeat() != nil && !glog.V(1) {
			return // Log heartbeats with verbose logging only.
		}
		var respTxt string
		var err error
		if *jsonFlag {
			respTxt, err = openconfig.SubscribeResponseToJSON(resp)
			if err != nil {
				glog.Fatal(err)
			}
		} else {
			respTxt = proto.MarshalTextString(resp)
		}
		fmt.Println(respTxt)
	}

	wg := new(sync.WaitGroup)
	for _, addr := range addrs {
		wg.Add(1)
		c := client.New(username, password, addr, opts)
		go c.Subscribe(wg, subscriptions, publish)
	}
	wg.Wait()
}
Exemplo n.º 5
0
func pushToOpenTSDB(addr string, conn OpenTSDBConn, config *Config,
	notif *openconfig.Notification) {

	if notif.Timestamp <= 0 {
		glog.Fatalf("Invalid timestamp %d in %s", notif.Timestamp, notif)
	}

	host := addr[:strings.IndexRune(addr, ':')]
	prefix := "/" + strings.Join(notif.Prefix.Element, "/")
	for _, update := range notif.Update {
		if update.Value == nil || update.Value.Type != openconfig.Type_JSON {
			glog.V(9).Infof("Ignoring incompatible update value in %s", update)
			continue
		}

		value := parseValue(update)
		if value == nil {
			glog.V(9).Infof("Ignoring non-numeric value in %s", update)
			continue
		}

		path := prefix + "/" + strings.Join(update.Path.Element, "/")
		metricName, tags := config.Match(path)
		if metricName == "" {
			glog.V(8).Infof("Ignoring unmatched update at %s: %+v", path, update.Value)
			continue
		}
		tags["host"] = host

		conn.Put(&DataPoint{
			Metric:    metricName,
			Timestamp: uint64(notif.Timestamp),
			Value:     value,
			Tags:      tags,
		})
	}
}
Exemplo n.º 6
0
// Subscribe sends subscriptions, and consumes responses.
// The given publish function is used to publish SubscribeResponses received
// for the given subscriptions, when connected to the given host, with the
// given user/pass pair, or the client-side cert specified in the gRPC opts.
// This function does not normally return so it should probably be run in its
// own goroutine.  When this function returns, the given WaitGroup is marked
// as done.
func (c *Client) Subscribe(wg *sync.WaitGroup, subscriptions []string,
	publish PublishFunc) {
	defer wg.Done()
	stream, err := c.client.Subscribe(c.ctx)
	if err != nil {
		glog.Fatalf("Subscribe failed: %s", err)
	}
	defer stream.CloseSend()

	for _, path := range subscriptions {
		sub := &openconfig.SubscribeRequest{
			Request: &openconfig.SubscribeRequest_Subscribe{
				Subscribe: &openconfig.SubscriptionList{
					Subscription: []*openconfig.Subscription{
						&openconfig.Subscription{
							Path: &openconfig.Path{Element: strings.Split(path, "/")},
						},
					},
				},
			},
		}

		glog.Infof("Sending subscribe request: %s", sub)
		err = stream.Send(sub)
		if err != nil {
			glog.Fatalf("Failed to subscribe: %s", err)
		}
	}

	for {
		resp, err := stream.Recv()
		if err != nil {
			if err != io.EOF {
				glog.Fatalf("Error received from the server: %s", err)
			}
			return
		}
		switch resp := resp.Response.(type) {
		case *openconfig.SubscribeResponse_SyncResponse:
			if !resp.SyncResponse {
				panic("initial sync failed," +
					" check that you're using a client compatible with the server")
			}
		}
		glog.V(3).Info(resp)
		publish(c.device, resp)
	}
}