func New(kfkConn sarama.Client) (*TransmitHandler, error) {
producer, err := sarama.NewSyncProducerFromClient(kfkConn)
if err != nil {
return nil, err
}
th := &TransmitHandler{
kill: killchan.New(),
dead: killchan.New(),
bus: make(chan Transmit),
producer: producer,
}
go th.run()
return th, nil
}
func New(peers []string, logger *log.Logger) (*ZooHandler, error) {
conn, _, err := zk.Connect(peers, time.Second)
if err != nil {
return nil, fmt.Errorf("Failed to connect to zookeeper: %v", err)
}
zh := &ZooHandler{
kill: killchan.New(),
dead: killchan.New(),
conn: conn,
logger: logger,
}
go zh.run()
return zh, nil
}
func New(kfkConn sarama.Client, transmitter Transmitter) (*ListenHandler, error) {
consumer, err := sarama.NewConsumerFromClient(kfkConn)
if err != nil {
return nil, fmt.Errorf("Failed to create Kafka consumer: %v", err)
}
lh := &ListenHandler{
listenbus: make(chan chan<- listenRequest),
transmitter: transmitter,
consumer: consumer,
kill: killchan.New(),
dead: killchan.New(),
}
go lh.run()
return lh, nil
}
func (lh *ListenHandler) listen(consumers map[Stream]listener, stream Stream, offset int64) error {
_, ok := consumers[stream]
if ok {
return ErrAlreadyListening
} else {
conn, err := lh.consumer.ConsumePartition(stream.Topic, stream.Partition, offset)
if err != nil {
return err
}
l := listener{
stream: stream,
conn: conn,
killed: killchan.New(),
}
consumers[stream] = l
go func() {
defer lh.sendListenRequest(stream, 0, nil, false)
defer l.killed.Kill()
for msg := range conn.Messages() {
lh.transmitter.Transmit(msg.Key, msg.Value, stream.Topic, stream.Partition, msg.Offset)
}
}()
}
return nil
}
func main() {
flag.Parse()
if *fPeers == "" {
flagbad("-peers is empty\n")
}
if *fPartition == -1 {
flagbad("-partition is empty\n")
}
if *fTopic == "" {
flagbad("-topic is empty\n")
}
var offset int64
switch *fOffset {
case "newest":
offset = sarama.OffsetNewest
case "oldest":
offset = sarama.OffsetOldest
case "manual":
offset = *fManual
if offset <= -1 {
flagbad("-manual positive integer expected")
}
default:
flagbad("-offset expects newest, oldest or time\n")
}
kfk, err := kafka.New("kafka-consumer", logger, strings.Split(*fPeers, ","))
if err != nil {
logger.Panicf("Failed to create kafka client: %v", err)
}
defer kfk.Close()
err = kfk.Listen(*fTopic, int32(*fPartition), offset)
if err != nil {
logger.Panicf("Failed to listen to partition %s:%d: %v", *fTopic, *fPartition, err)
}
sigchan := make(chan os.Signal)
signal.Notify(sigchan, syscall.SIGINT)
kc := killchan.New()
go func() {
<-sigchan
kc.Kill()
}()
for {
select {
case <-kc.Chan():
logger.Printf("Interrupt\n")
return
case message := <-kfk.Incoming():
if *fVerbose {
fmt.Printf("%s:%d (offset %d) %s\n", message.Topic, message.Partition, message.Offset, message.Val)
} else {
fmt.Printf("%s\n", message.Val)
}
}
}
}