// NewClient returns a Kafka client
func NewClient(addresses []string) (sarama.Client, error) {
config := sarama.NewConfig()
hostname, err := os.Hostname()
if err != nil {
hostname = ""
}
config.ClientID = hostname
config.Producer.Compression = sarama.CompressionSnappy
config.Producer.Return.Successes = true
var client sarama.Client
retries := outOfBrokersRetries + 1
for retries > 0 {
client, err = sarama.NewClient(addresses, config)
retries--
if err == sarama.ErrOutOfBrokers {
glog.Errorf("Can't connect to the Kafka cluster at %s (%d retries left): %s",
addresses, retries, err)
time.Sleep(outOfBrokersBackoff)
} else {
break
}
}
return client, err
}
// Run sets up the HTTP server and any handlers
func (s *server) Run() {
http.HandleFunc("/debug", debugHandler)
http.HandleFunc("/debug/latency", latencyHandler)
// monitoring server
err := http.ListenAndServe(s.serverName, nil)
if err != nil {
glog.Errorf("Could not start monitor server: %s", err)
}
}
// NewServer creates a new server struct
func NewServer(address string) Server {
vrfName, addr, err := netns.ParseAddress(address)
if err != nil {
glog.Errorf("Failed to parse address: %s", err)
}
return &server{
vrfName: vrfName,
serverName: addr,
}
}
// handleErrors reads from the producer's errors channel and collects some information
// for monitoring
func (p *producer) handleErrors() {
defer p.wg.Done()
for msg := range p.kafkaProducer.Errors() {
metadata := msg.Msg.Metadata.(kafka.Metadata)
// TODO: Add a monotonic clock source when one becomes available
p.histogram.UpdateLatencyValues(metadata.StartTime, time.Now())
glog.Errorf("Kafka Producer error: %s", msg.Error())
p.numFailures.Add(uint64(metadata.NumMessages))
}
}
func main() {
tsdbFlag := flag.String("tsdb", "",
"Address of the OpenTSDB server where to push telemetry to")
textFlag := flag.Bool("text", false,
"Print the output as simple text")
configFlag := flag.String("config", "",
"Config to turn OpenConfig telemetry into OpenTSDB put requests")
username, password, subscriptions, addrs, opts := client.ParseFlags()
if !(*tsdbFlag != "" || *textFlag) {
glog.Fatal("Specify the address of the OpenTSDB server to write to with -tsdb")
} else if *configFlag == "" {
glog.Fatal("Specify a JSON configuration file with -config")
}
config, err := loadConfig(*configFlag)
if err != nil {
glog.Fatal(err)
}
// Ignore the default "subscribe-to-everything" subscription of the
// -subscribe flag.
if subscriptions[0] == "" {
subscriptions = subscriptions[1:]
}
// Add the subscriptions from the config file.
subscriptions = append(subscriptions, config.Subscriptions...)
var c OpenTSDBConn
if *textFlag {
c = newTextDumper()
} else {
// TODO: support HTTP(S).
c = newTelnetClient(*tsdbFlag)
}
wg := new(sync.WaitGroup)
for _, addr := range addrs {
wg.Add(1)
publish := func(addr string, message proto.Message) {
resp, ok := message.(*openconfig.SubscribeResponse)
if !ok {
glog.Errorf("Unexpected type of message: %T", message)
return
}
if notif := resp.GetUpdate(); notif != nil {
pushToOpenTSDB(addr, c, config, notif)
}
}
c := client.New(username, password, addr, opts)
go c.Subscribe(wg, subscriptions, publish)
}
wg.Wait()
}
func main() {
username, password, subscriptions, hostAddrs, opts := occlient.ParseFlags()
if *redisFlag == "" {
glog.Fatal("Specify the address of the Redis server to write to with -redis")
}
redisAddrs := strings.Split(*redisFlag, ",")
if !*clusterMode && len(redisAddrs) > 1 {
glog.Fatal("Please pass only 1 redis address in noncluster mode or enable cluster mode")
}
if *clusterMode {
client = redis.NewClusterClient(&redis.ClusterOptions{
Addrs: redisAddrs,
Password: *redisPassword,
})
} else {
client = redis.NewClient(&redis.Options{
Addr: *redisFlag,
Password: *redisPassword,
})
}
defer client.Close()
// TODO: Figure out ways to handle being in the wrong mode:
// Connecting to cluster in non cluster mode - we get a MOVED error on the first HMSET
// Connecting to a noncluster in cluster mode - we get stuck forever
_, err := client.Ping().Result()
if err != nil {
glog.Fatal("Failed to connect to client: ", err)
}
ocPublish := func(addr string, message proto.Message) {
resp, ok := message.(*openconfig.SubscribeResponse)
if !ok {
glog.Errorf("Unexpected type of message: %T", message)
return
}
if notif := resp.GetUpdate(); notif != nil {
bufferToRedis(addr, notif)
}
}
wg := new(sync.WaitGroup)
for _, hostAddr := range hostAddrs {
wg.Add(1)
c := occlient.New(username, password, hostAddr, opts)
go c.Subscribe(wg, subscriptions, ocPublish)
}
wg.Wait()
}
func main() {
username, password, subscriptions, addrs, opts := client.ParseFlags()
if *getFlag != "" {
c := client.New(username, password, addrs[0], opts)
for _, notification := range c.Get(*getFlag) {
var notifStr string
if *jsonFlag {
var err error
if notifStr, err = openconfig.NotificationToJSON(notification); err != nil {
glog.Fatal(err)
}
} else {
notifStr = notification.String()
}
fmt.Println(notifStr)
}
return
}
publish := func(addr string, message proto.Message) {
resp, ok := message.(*pb.SubscribeResponse)
if !ok {
glog.Errorf("Unexpected type of message: %T", message)
return
}
if resp.GetHeartbeat() != nil && !glog.V(1) {
return // Log heartbeats with verbose logging only.
}
var respTxt string
var err error
if *jsonFlag {
respTxt, err = openconfig.SubscribeResponseToJSON(resp)
if err != nil {
glog.Fatal(err)
}
} else {
respTxt = proto.MarshalTextString(resp)
}
fmt.Println(respTxt)
}
wg := new(sync.WaitGroup)
for _, addr := range addrs {
wg.Add(1)
c := client.New(username, password, addr, opts)
go c.Subscribe(wg, subscriptions, publish)
}
wg.Wait()
}
func (c *client) Run(ch chan<- *pb.LanzRecord) {
for !c.stopping {
if err := c.conn.Connect(); err != nil && !c.stopping {
glog.V(1).Infof("Can't connect to LANZ server: %v", err)
time.Sleep(c.backoff)
continue
}
glog.V(1).Infof("Connected successfully to LANZ server: %v", c.addr)
if err := c.read(bufio.NewReader(c.conn), ch); err != nil && !c.stopping {
if err != io.EOF && err != io.ErrUnexpectedEOF {
glog.Errorf("Error receiving LANZ events: %v", err)
}
c.conn.Close()
time.Sleep(c.backoff)
}
}
close(ch)
}