Ejemplo n.º 1
0
// NewClient returns a Kafka client
func NewClient(addresses []string) (sarama.Client, error) {
	config := sarama.NewConfig()
	hostname, err := os.Hostname()
	if err != nil {
		hostname = ""
	}
	config.ClientID = hostname
	config.Producer.Compression = sarama.CompressionSnappy
	config.Producer.Return.Successes = true

	var client sarama.Client
	retries := outOfBrokersRetries + 1
	for retries > 0 {
		client, err = sarama.NewClient(addresses, config)
		retries--
		if err == sarama.ErrOutOfBrokers {
			glog.Errorf("Can't connect to the Kafka cluster at %s (%d retries left): %s",
				addresses, retries, err)
			time.Sleep(outOfBrokersBackoff)
		} else {
			break
		}
	}
	return client, err
}
Ejemplo n.º 2
0
// Run sets up the HTTP server and any handlers
func (s *server) Run() {
	http.HandleFunc("/debug", debugHandler)
	http.HandleFunc("/debug/latency", latencyHandler)

	// monitoring server
	err := http.ListenAndServe(s.serverName, nil)
	if err != nil {
		glog.Errorf("Could not start monitor server: %s", err)
	}
}
Ejemplo n.º 3
0
// NewServer creates a new server struct
func NewServer(address string) Server {
	vrfName, addr, err := netns.ParseAddress(address)
	if err != nil {
		glog.Errorf("Failed to parse address: %s", err)
	}
	return &server{
		vrfName:    vrfName,
		serverName: addr,
	}
}
Ejemplo n.º 4
0
// handleErrors reads from the producer's errors channel and collects some information
// for monitoring
func (p *producer) handleErrors() {
	defer p.wg.Done()
	for msg := range p.kafkaProducer.Errors() {
		metadata := msg.Msg.Metadata.(kafka.Metadata)
		// TODO: Add a monotonic clock source when one becomes available
		p.histogram.UpdateLatencyValues(metadata.StartTime, time.Now())
		glog.Errorf("Kafka Producer error: %s", msg.Error())
		p.numFailures.Add(uint64(metadata.NumMessages))
	}
}
Ejemplo n.º 5
0
func main() {
	tsdbFlag := flag.String("tsdb", "",
		"Address of the OpenTSDB server where to push telemetry to")
	textFlag := flag.Bool("text", false,
		"Print the output as simple text")
	configFlag := flag.String("config", "",
		"Config to turn OpenConfig telemetry into OpenTSDB put requests")
	username, password, subscriptions, addrs, opts := client.ParseFlags()

	if !(*tsdbFlag != "" || *textFlag) {
		glog.Fatal("Specify the address of the OpenTSDB server to write to with -tsdb")
	} else if *configFlag == "" {
		glog.Fatal("Specify a JSON configuration file with -config")
	}

	config, err := loadConfig(*configFlag)
	if err != nil {
		glog.Fatal(err)
	}
	// Ignore the default "subscribe-to-everything" subscription of the
	// -subscribe flag.
	if subscriptions[0] == "" {
		subscriptions = subscriptions[1:]
	}
	// Add the subscriptions from the config file.
	subscriptions = append(subscriptions, config.Subscriptions...)

	var c OpenTSDBConn
	if *textFlag {
		c = newTextDumper()
	} else {
		// TODO: support HTTP(S).
		c = newTelnetClient(*tsdbFlag)
	}

	wg := new(sync.WaitGroup)
	for _, addr := range addrs {
		wg.Add(1)
		publish := func(addr string, message proto.Message) {
			resp, ok := message.(*openconfig.SubscribeResponse)
			if !ok {
				glog.Errorf("Unexpected type of message: %T", message)
				return
			}
			if notif := resp.GetUpdate(); notif != nil {
				pushToOpenTSDB(addr, c, config, notif)
			}
		}
		c := client.New(username, password, addr, opts)
		go c.Subscribe(wg, subscriptions, publish)
	}
	wg.Wait()
}
Ejemplo n.º 6
0
func main() {
	username, password, subscriptions, hostAddrs, opts := occlient.ParseFlags()
	if *redisFlag == "" {
		glog.Fatal("Specify the address of the Redis server to write to with -redis")
	}

	redisAddrs := strings.Split(*redisFlag, ",")
	if !*clusterMode && len(redisAddrs) > 1 {
		glog.Fatal("Please pass only 1 redis address in noncluster mode or enable cluster mode")
	}

	if *clusterMode {
		client = redis.NewClusterClient(&redis.ClusterOptions{
			Addrs:    redisAddrs,
			Password: *redisPassword,
		})
	} else {
		client = redis.NewClient(&redis.Options{
			Addr:     *redisFlag,
			Password: *redisPassword,
		})
	}
	defer client.Close()

	// TODO: Figure out ways to handle being in the wrong mode:
	// Connecting to cluster in non cluster mode - we get a MOVED error on the first HMSET
	// Connecting to a noncluster in cluster mode - we get stuck forever
	_, err := client.Ping().Result()
	if err != nil {
		glog.Fatal("Failed to connect to client: ", err)
	}

	ocPublish := func(addr string, message proto.Message) {
		resp, ok := message.(*openconfig.SubscribeResponse)
		if !ok {
			glog.Errorf("Unexpected type of message: %T", message)
			return
		}
		if notif := resp.GetUpdate(); notif != nil {
			bufferToRedis(addr, notif)
		}
	}

	wg := new(sync.WaitGroup)
	for _, hostAddr := range hostAddrs {
		wg.Add(1)
		c := occlient.New(username, password, hostAddr, opts)
		go c.Subscribe(wg, subscriptions, ocPublish)
	}
	wg.Wait()
}
Ejemplo n.º 7
0
func main() {
	username, password, subscriptions, addrs, opts := client.ParseFlags()

	if *getFlag != "" {
		c := client.New(username, password, addrs[0], opts)
		for _, notification := range c.Get(*getFlag) {
			var notifStr string
			if *jsonFlag {
				var err error
				if notifStr, err = openconfig.NotificationToJSON(notification); err != nil {
					glog.Fatal(err)
				}
			} else {
				notifStr = notification.String()
			}
			fmt.Println(notifStr)

		}
		return
	}

	publish := func(addr string, message proto.Message) {
		resp, ok := message.(*pb.SubscribeResponse)
		if !ok {
			glog.Errorf("Unexpected type of message: %T", message)
			return
		}
		if resp.GetHeartbeat() != nil && !glog.V(1) {
			return // Log heartbeats with verbose logging only.
		}
		var respTxt string
		var err error
		if *jsonFlag {
			respTxt, err = openconfig.SubscribeResponseToJSON(resp)
			if err != nil {
				glog.Fatal(err)
			}
		} else {
			respTxt = proto.MarshalTextString(resp)
		}
		fmt.Println(respTxt)
	}

	wg := new(sync.WaitGroup)
	for _, addr := range addrs {
		wg.Add(1)
		c := client.New(username, password, addr, opts)
		go c.Subscribe(wg, subscriptions, publish)
	}
	wg.Wait()
}
Ejemplo n.º 8
0
func (c *client) Run(ch chan<- *pb.LanzRecord) {
	for !c.stopping {
		if err := c.conn.Connect(); err != nil && !c.stopping {
			glog.V(1).Infof("Can't connect to LANZ server: %v", err)
			time.Sleep(c.backoff)
			continue
		}
		glog.V(1).Infof("Connected successfully to LANZ server: %v", c.addr)
		if err := c.read(bufio.NewReader(c.conn), ch); err != nil && !c.stopping {
			if err != io.EOF && err != io.ErrUnexpectedEOF {
				glog.Errorf("Error receiving LANZ events: %v", err)
			}
			c.conn.Close()
			time.Sleep(c.backoff)
		}
	}

	close(ch)
}