// NewClient returns a Kafka client func NewClient(addresses []string) (sarama.Client, error) { config := sarama.NewConfig() hostname, err := os.Hostname() if err != nil { hostname = "" } config.ClientID = hostname config.Producer.Compression = sarama.CompressionSnappy config.Producer.Return.Successes = true var client sarama.Client retries := outOfBrokersRetries + 1 for retries > 0 { client, err = sarama.NewClient(addresses, config) retries-- if err == sarama.ErrOutOfBrokers { glog.Errorf("Can't connect to the Kafka cluster at %s (%d retries left): %s", addresses, retries, err) time.Sleep(outOfBrokersBackoff) } else { break } } return client, err }
// Run sets up the HTTP server and any handlers func (s *server) Run() { http.HandleFunc("/debug", debugHandler) http.HandleFunc("/debug/latency", latencyHandler) // monitoring server err := http.ListenAndServe(s.serverName, nil) if err != nil { glog.Errorf("Could not start monitor server: %s", err) } }
// NewServer creates a new server struct func NewServer(address string) Server { vrfName, addr, err := netns.ParseAddress(address) if err != nil { glog.Errorf("Failed to parse address: %s", err) } return &server{ vrfName: vrfName, serverName: addr, } }
// handleErrors reads from the producer's errors channel and collects some information // for monitoring func (p *producer) handleErrors() { defer p.wg.Done() for msg := range p.kafkaProducer.Errors() { metadata := msg.Msg.Metadata.(kafka.Metadata) // TODO: Add a monotonic clock source when one becomes available p.histogram.UpdateLatencyValues(metadata.StartTime, time.Now()) glog.Errorf("Kafka Producer error: %s", msg.Error()) p.numFailures.Add(uint64(metadata.NumMessages)) } }
func main() { tsdbFlag := flag.String("tsdb", "", "Address of the OpenTSDB server where to push telemetry to") textFlag := flag.Bool("text", false, "Print the output as simple text") configFlag := flag.String("config", "", "Config to turn OpenConfig telemetry into OpenTSDB put requests") username, password, subscriptions, addrs, opts := client.ParseFlags() if !(*tsdbFlag != "" || *textFlag) { glog.Fatal("Specify the address of the OpenTSDB server to write to with -tsdb") } else if *configFlag == "" { glog.Fatal("Specify a JSON configuration file with -config") } config, err := loadConfig(*configFlag) if err != nil { glog.Fatal(err) } // Ignore the default "subscribe-to-everything" subscription of the // -subscribe flag. if subscriptions[0] == "" { subscriptions = subscriptions[1:] } // Add the subscriptions from the config file. subscriptions = append(subscriptions, config.Subscriptions...) var c OpenTSDBConn if *textFlag { c = newTextDumper() } else { // TODO: support HTTP(S). c = newTelnetClient(*tsdbFlag) } wg := new(sync.WaitGroup) for _, addr := range addrs { wg.Add(1) publish := func(addr string, message proto.Message) { resp, ok := message.(*openconfig.SubscribeResponse) if !ok { glog.Errorf("Unexpected type of message: %T", message) return } if notif := resp.GetUpdate(); notif != nil { pushToOpenTSDB(addr, c, config, notif) } } c := client.New(username, password, addr, opts) go c.Subscribe(wg, subscriptions, publish) } wg.Wait() }
func main() { username, password, subscriptions, hostAddrs, opts := occlient.ParseFlags() if *redisFlag == "" { glog.Fatal("Specify the address of the Redis server to write to with -redis") } redisAddrs := strings.Split(*redisFlag, ",") if !*clusterMode && len(redisAddrs) > 1 { glog.Fatal("Please pass only 1 redis address in noncluster mode or enable cluster mode") } if *clusterMode { client = redis.NewClusterClient(&redis.ClusterOptions{ Addrs: redisAddrs, Password: *redisPassword, }) } else { client = redis.NewClient(&redis.Options{ Addr: *redisFlag, Password: *redisPassword, }) } defer client.Close() // TODO: Figure out ways to handle being in the wrong mode: // Connecting to cluster in non cluster mode - we get a MOVED error on the first HMSET // Connecting to a noncluster in cluster mode - we get stuck forever _, err := client.Ping().Result() if err != nil { glog.Fatal("Failed to connect to client: ", err) } ocPublish := func(addr string, message proto.Message) { resp, ok := message.(*openconfig.SubscribeResponse) if !ok { glog.Errorf("Unexpected type of message: %T", message) return } if notif := resp.GetUpdate(); notif != nil { bufferToRedis(addr, notif) } } wg := new(sync.WaitGroup) for _, hostAddr := range hostAddrs { wg.Add(1) c := occlient.New(username, password, hostAddr, opts) go c.Subscribe(wg, subscriptions, ocPublish) } wg.Wait() }
func main() { username, password, subscriptions, addrs, opts := client.ParseFlags() if *getFlag != "" { c := client.New(username, password, addrs[0], opts) for _, notification := range c.Get(*getFlag) { var notifStr string if *jsonFlag { var err error if notifStr, err = openconfig.NotificationToJSON(notification); err != nil { glog.Fatal(err) } } else { notifStr = notification.String() } fmt.Println(notifStr) } return } publish := func(addr string, message proto.Message) { resp, ok := message.(*pb.SubscribeResponse) if !ok { glog.Errorf("Unexpected type of message: %T", message) return } if resp.GetHeartbeat() != nil && !glog.V(1) { return // Log heartbeats with verbose logging only. } var respTxt string var err error if *jsonFlag { respTxt, err = openconfig.SubscribeResponseToJSON(resp) if err != nil { glog.Fatal(err) } } else { respTxt = proto.MarshalTextString(resp) } fmt.Println(respTxt) } wg := new(sync.WaitGroup) for _, addr := range addrs { wg.Add(1) c := client.New(username, password, addr, opts) go c.Subscribe(wg, subscriptions, publish) } wg.Wait() }
func (c *client) Run(ch chan<- *pb.LanzRecord) { for !c.stopping { if err := c.conn.Connect(); err != nil && !c.stopping { glog.V(1).Infof("Can't connect to LANZ server: %v", err) time.Sleep(c.backoff) continue } glog.V(1).Infof("Connected successfully to LANZ server: %v", c.addr) if err := c.read(bufio.NewReader(c.conn), ch); err != nil && !c.stopping { if err != io.EOF && err != io.ErrUnexpectedEOF { glog.Errorf("Error receiving LANZ events: %v", err) } c.conn.Close() time.Sleep(c.backoff) } } close(ch) }