Example #1
0
func New(kfkConn sarama.Client) (*TransmitHandler, error) {
	producer, err := sarama.NewSyncProducerFromClient(kfkConn)
	if err != nil {
		return nil, err
	}
	th := &TransmitHandler{
		kill:     killchan.New(),
		dead:     killchan.New(),
		bus:      make(chan Transmit),
		producer: producer,
	}
	go th.run()
	return th, nil
}
Example #2
0
func New(peers []string, logger *log.Logger) (*ZooHandler, error) {
	conn, _, err := zk.Connect(peers, time.Second)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to zookeeper: %v", err)
	}
	zh := &ZooHandler{
		kill:   killchan.New(),
		dead:   killchan.New(),
		conn:   conn,
		logger: logger,
	}
	go zh.run()
	return zh, nil
}
Example #3
0
func New(kfkConn sarama.Client, transmitter Transmitter) (*ListenHandler, error) {
	consumer, err := sarama.NewConsumerFromClient(kfkConn)
	if err != nil {
		return nil, fmt.Errorf("Failed to create Kafka consumer: %v", err)
	}
	lh := &ListenHandler{
		listenbus:   make(chan chan<- listenRequest),
		transmitter: transmitter,
		consumer:    consumer,
		kill:        killchan.New(),
		dead:        killchan.New(),
	}
	go lh.run()
	return lh, nil
}
Example #4
0
func (lh *ListenHandler) listen(consumers map[Stream]listener, stream Stream, offset int64) error {
	_, ok := consumers[stream]
	if ok {
		return ErrAlreadyListening
	} else {
		conn, err := lh.consumer.ConsumePartition(stream.Topic, stream.Partition, offset)
		if err != nil {
			return err
		}
		l := listener{
			stream: stream,
			conn:   conn,
			killed: killchan.New(),
		}
		consumers[stream] = l
		go func() {
			defer lh.sendListenRequest(stream, 0, nil, false)
			defer l.killed.Kill()
			for msg := range conn.Messages() {
				lh.transmitter.Transmit(msg.Key, msg.Value, stream.Topic, stream.Partition, msg.Offset)
			}
		}()
	}
	return nil
}
Example #5
0
func main() {
	flag.Parse()
	if *fPeers == "" {
		flagbad("-peers is empty\n")
	}
	if *fPartition == -1 {
		flagbad("-partition is empty\n")
	}
	if *fTopic == "" {
		flagbad("-topic is empty\n")
	}
	var offset int64
	switch *fOffset {
	case "newest":
		offset = sarama.OffsetNewest
	case "oldest":
		offset = sarama.OffsetOldest
	case "manual":
		offset = *fManual
		if offset <= -1 {
			flagbad("-manual positive integer expected")
		}
	default:
		flagbad("-offset expects newest, oldest or time\n")
	}

	kfk, err := kafka.New("kafka-consumer", logger, strings.Split(*fPeers, ","))
	if err != nil {
		logger.Panicf("Failed to create kafka client: %v", err)
	}
	defer kfk.Close()

	err = kfk.Listen(*fTopic, int32(*fPartition), offset)
	if err != nil {
		logger.Panicf("Failed to listen to partition %s:%d: %v", *fTopic, *fPartition, err)
	}

	sigchan := make(chan os.Signal)
	signal.Notify(sigchan, syscall.SIGINT)
	kc := killchan.New()
	go func() {
		<-sigchan
		kc.Kill()
	}()

	for {
		select {
		case <-kc.Chan():
			logger.Printf("Interrupt\n")
			return
		case message := <-kfk.Incoming():
			if *fVerbose {
				fmt.Printf("%s:%d (offset %d) %s\n", message.Topic, message.Partition, message.Offset, message.Val)
			} else {
				fmt.Printf("%s\n", message.Val)
			}
		}
	}
}