func generateKafkaData(t *testing.T, topic string) {
config := sarama.NewConfig()
client, err := sarama.NewClient([]string{getTestKafkaHost()}, config)
if err != nil {
t.Errorf("%s", err)
}
producer, err := sarama.NewSyncProducerFromClient(client)
if err != nil {
t.Error(err)
}
defer producer.Close()
msg := &sarama.ProducerMessage{
Topic: topic,
Value: sarama.StringEncoder("Hello World"),
}
_, _, err = producer.SendMessage(msg)
if err != nil {
t.Errorf("FAILED to send message: %s\n", err)
}
client.RefreshMetadata(topic)
}
func getProducer(client sarama.Client) sarama.SyncProducer {
//fmt.Println("Getting producer.")
producer, err := sarama.NewSyncProducerFromClient(client)
if err != nil {
panic(err)
}
return producer
}
// clientProducer generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// Default 5 instances of clientProducer are created under each Kafka client.
func clientProducer(c kafka.Client) {
producer, err := kafka.NewSyncProducerFromClient(c)
if err != nil {
log.Println(err.Error())
}
defer producer.Close()
// Instantiate rand per producer to avoid mutex contention.
source := rand.NewSource(time.Now().UnixNano())
generator := rand.New(source)
msgData := make([]byte, msgSize)
// Use a local accumulator then periodically update global counter.
// Global counter can become a bottleneck with too many threads.
tick := time.Tick(3 * time.Millisecond)
var n int64
for {
// Message rate limit works by having all clientProducer loops incrementing
// a global counter and tracking the aggregate per-second progress.
// If the configured rate is met, the worker will sleep
// for the remainder of the 1 second window.
rateEnd := time.Now().Add(time.Second)
countStart := fetchSent()
var start time.Time
for fetchSent()-countStart < msgRate {
randMsg(msgData, *generator)
msg := &kafka.ProducerMessage{Topic: topic, Value: kafka.ByteEncoder(msgData)}
// We start timing after the message is created.
// This ensures latency metering from the time between message sent and receiving an ack.
start = time.Now()
_, _, err = producer.SendMessage(msg)
if err != nil {
log.Println(err)
} else {
// Increment global sent count and fire off time since start value into the latency channel.
n++
select {
case <-tick:
incrSent(n)
n = 0
default:
break
}
latencies <- time.Since(start).Seconds() * 1000
}
}
// If the global per-second rate limit was met,
// the inner loop breaks and the outer loop sleeps for the second remainder.
time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
}
}
func New(kfkConn sarama.Client) (*TransmitHandler, error) {
producer, err := sarama.NewSyncProducerFromClient(kfkConn)
if err != nil {
return nil, err
}
th := &TransmitHandler{
kill: killchan.New(),
dead: killchan.New(),
bus: make(chan Transmit),
producer: producer,
}
go th.run()
return th, nil
}
// Seed messages
func testSeed(n int) error {
producer, err := sarama.NewSyncProducerFromClient(testClient)
if err != nil {
return err
}
for i := 0; i < n; i++ {
kv := sarama.StringEncoder(fmt.Sprintf("PLAINDATA-%08d", i))
for _, t := range testTopics {
msg := &sarama.ProducerMessage{Topic: t, Key: kv, Value: kv}
if _, _, err := producer.SendMessage(msg); err != nil {
return err
}
}
}
return producer.Close()
}
// NewKafkaDeliver creates connection to Kafka
func NewKafkaDeliver(store *Store, clientID string, brokerList []string) (*KafkaDeliver, error) {
config := sarama.NewConfig()
config.ClientID = clientID
config.Producer.RequiredAcks = sarama.WaitForAll
config.Net.DialTimeout = 2 * time.Second // real connect time is about 4* DialTimeout (?)
client, err := sarama.NewClient(brokerList, config)
if err != nil {
return nil, err
}
producer, err := sarama.NewSyncProducerFromClient(client)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
log.Errorf("go=kafka at=defer-close-producer after error %v\n", err)
if err := producer.Close(); err != nil {
log.Errorf("go=kafka at=producer-close fatal error %v\n", err)
os.Exit(4)
}
}
}()
return &KafkaDeliver{
clientID: clientID,
brokerList: brokerList,
store: store,
producer: producer,
client: client,
config: config,
deliverGoroutines: maxDeliverGoroutines,
shutdownDeliver: make(chan bool, maxDeliverGoroutines),
shutdown: make(chan bool, maxDeliverGoroutines),
}, nil
}
func (k *kBroker) Connect() error {
if k.c != nil {
return nil
}
c, err := sarama.NewClient(k.addrs, sarama.NewConfig())
if err != nil {
return err
}
k.c = c
p, err := sarama.NewSyncProducerFromClient(c)
if err != nil {
return err
}
k.p = p
config := sc.NewConfig()
// TODO: make configurable offset as SubscriberOption
config.Config.Consumer.Offsets.Initial = sarama.OffsetNewest
cs, err := sc.NewClient(k.addrs, config)
if err != nil {
return err
}
k.sc = cs
// TODO: TLS
/*
opts.Secure = k.opts.Secure
opts.TLSConfig = k.opts.TLSConfig
// secure might not be set
if k.opts.TLSConfig != nil {
opts.Secure = true
}
*/
return nil
}
// NewPeer creates and returns a new Peer for communicating with Kafka.
func NewPeer(host string) (*Peer, error) {
sarama.Logger = log.New(os.Stdout, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile)
host = strings.Split(host, ":")[0] + ":9092"
config := sarama.NewConfig()
config.Consumer.Fetch.Default = 10 * 1024 * 1024
client, err := sarama.NewClient([]string{host}, config)
if err != nil {
return nil, err
}
producer, err := sarama.NewSyncProducerFromClient(client)
if err != nil {
return nil, err
}
return &Peer{
client: client,
producer: producer,
send: make(chan []byte),
errors: make(chan error, 1),
done: make(chan bool),
}, nil
}
func newProducerFromZookeeper() (sarama.Client, sarama.SyncProducer, error) {
brokers, err := kafkazk.LookupBrokers(cfg.zkstring)
if err != nil {
return nil, nil, err
}
brokerStr := make([]string, len(brokers))
for i, b := range brokers {
brokerStr[i] = fmt.Sprintf("%s:%d", b.Host, b.Port)
}
logger.Println("connecting to Kafka, using brokers from ZooKeeper:", brokerStr)
client, err := sarama.NewClient(brokerStr, sarama.NewConfig())
if err != nil {
return nil, nil, err
}
producer, err := sarama.NewSyncProducerFromClient(client)
if err != nil {
return nil, nil, err
}
return client, producer, nil
}
func main() {
var proc = flag.Int("proc", 16, "max concurency")
var srv = flag.String("srv", "0.0.0.0:8004", "server socket")
var netprofile = flag.Bool("netprofile", false, "open socket for remote profiling")
var brokers = flag.String("brokers", "./brokers", "file containing Kafka brokers to connect to")
flag.Parse()
if *netprofile {
go func() {
trace.AuthRequest = func(req *http.Request) (any, sensitive bool) {
return true, true
}
addr := "0.0.0.0:6061"
log.Println("start debug HTTP server", addr)
log.Println(http.ListenAndServe(addr, nil))
}()
}
statInit()
hostname, _ := os.Hostname()
log.Println("GOMAXPROCS", runtime.GOMAXPROCS(0))
log.Println("hostname", hostname)
brokerContent, err := ioutil.ReadFile(*brokers)
if err != nil {
log.Fatalf("failed to read file '%s': %v", brokers, err)
}
brokerList := strings.Split(string(brokerContent), "\n")
log.Printf("kafka brokers: %s", strings.Join(brokerList, ", "))
kafkaClient, err := newKafkaClient(*proc, brokerList, hostname)
if err != nil {
log.Fatalf("failed to connect to kafka: %v", err)
}
defer kafkaClient.Close()
dataProducer, _ := sarama.NewSyncProducerFromClient(kafkaClient)
defer dataProducer.Close()
indexProducer, _ := sarama.NewSyncProducerFromClient(kafkaClient)
defer indexProducer.Close()
wg := &sync.WaitGroup{}
tickCh := time.Tick(time.Second)
indexDumperCh := startIndexDumper(indexProducer, wg)
indexBuilderCh := startIndexBuilder(*proc, indexDumperCh, tickCh, wg)
log.Println("listen to", *srv)
conn, err := net.Listen("tcp", *srv)
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
grpcServer := grpc.NewServer(grpc.MaxConcurrentStreams(uint32(*proc * 2)))
pb.RegisterIndexerServer(grpcServer, &indexerServer{dataProducer, indexBuilderCh})
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-c
log.Println("stopping gRPC")
grpcServer.Stop()
}()
grpcServer.Serve(conn)
time.Sleep(100 * time.Millisecond) // let gRPC's goroutines to complete
close(indexBuilderCh)
log.Println("waiting completion of goroutines")
wg.Wait()
log.Println("bye")
}