Beispiel #1
0
func configure() config {
	if cfg != nil {
		return *cfg
	}

	cfg = new(config)
	if err := gofigure.Gofigure(cfg); err != nil {
		log.Error(err, nil)
		os.Exit(1)
	}

	log.Namespace = cfg.Namespace()

	return *cfg
}
Beispiel #2
0
func main() {
	var cmd *exec.Cmd
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		for {
			select {
			case sig := <-sigs:
				cmd.Process.Signal(sig)
			}
		}
	}()

	var cfg config
	if err := gofigure.Gofigure(&cfg); err != nil {
		fmt.Fprintln(os.Stderr, "unexpected error configuring ktee")
		os.Exit(1)
	}

	var err error
	var producer sarama.SyncProducer

	if len(cfg.Brokers) > 0 {
		brokers := strings.Split(cfg.Brokers, ",")
		producer, err = sarama.NewSyncProducer(brokers, sarama.NewConfig())
		if err != nil {
			fmt.Fprintf(os.Stderr, "error connecting to Kafka brokers: %s\n", err)
			os.Exit(1)
		}

		defer func() {
			producer.Close()
		}()
	}

	args := os.Args[1:]
	if len(args) == 0 {
		fmt.Fprintln(os.Stderr, "usage: ktee args")
		os.Exit(1)
	}

	kwOut := kafkaWriter{producer, os.Stdout, cfg.OutTopic, new(bytes.Buffer), make(chan sarama.ProducerMessage)}
	kwErr := kafkaWriter{producer, os.Stderr, cfg.ErrTopic, new(bytes.Buffer), make(chan sarama.ProducerMessage)}

	defer func() {
		kwOut.Flush()
		kwErr.Flush()
	}()

	cmd = exec.Command(args[0], args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = kwOut
	cmd.Stderr = kwErr
	cmd.Env = os.Environ()

	err = cmd.Run()
	if err != nil {
		switch err.(type) {
		case *exec.ExitError:
			fmt.Fprintf(os.Stderr, "non-zero exit code: %s\n", err)
			if status, ok := err.(*exec.ExitError).Sys().(syscall.WaitStatus); ok {
				os.Exit(status.ExitStatus())
			}
			os.Exit(1)
		default:
			fmt.Fprintf(os.Stderr, "error executing command: %s\n", err)
			os.Exit(1)
		}
	}
}
Beispiel #3
0
func main() {
	cfg = &config{
		Partitions: "all",
		Offset:     "newest",
	}
	err := gofigure.Gofigure(cfg)
	if err != nil {
		log.Fatal(err)
	}

	if cfg.BrokerList == "" {
		log.Fatal(errors.New("-broker-list or BROKER_LIST is required (as a comma-separated list)"))
	}

	if cfg.Topic == "" {
		log.Fatal(errors.New("-topic or TOPIC is required"))
	}

	var initialOffset int64
	switch cfg.Offset {
	case "oldest":
		initialOffset = sarama.OffsetOldest
	case "newest":
		initialOffset = sarama.OffsetNewest
	default:
		log.Fatal(errors.New("-offset/OFFSET should be `oldest` or `newest`"))
	}

	c, err := sarama.NewConsumer(strings.Split(cfg.BrokerList, ","), nil)
	if err != nil {
		log.Fatal(fmt.Errorf("Failed to start consumer: %s", err))
	}

	partitionList, err := getPartitions(c)
	if err != nil {
		log.Fatal(fmt.Errorf("Failed to get the list of partitions: %s", err))
	}

	var (
		messages = make(chan *sarama.ConsumerMessage, 256)
		closing  = make(chan struct{})
		wg       sync.WaitGroup
	)

	go func() {
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Kill, os.Interrupt)
		<-signals
		log.Println("Initiating shutdown of consumer...")
		close(closing)
	}()

	for _, partition := range partitionList {
		pc, err := c.ConsumePartition(cfg.Topic, partition, initialOffset)
		if err != nil {
			log.Fatal(fmt.Errorf("Failed to start consumer for partition %d: %s", partition, err))
		}

		go func(pc sarama.PartitionConsumer) {
			<-closing
			pc.AsyncClose()
		}(pc)

		wg.Add(1)
		go func(pc sarama.PartitionConsumer) {
			defer wg.Done()
			for message := range pc.Messages() {
				messages <- message
			}
		}(pc)
	}

	go func() {
		for msg := range messages {
			if len(msg.Value) > 0 {
				var m map[string]interface{}
				err := json.Unmarshal(msg.Value, &m)
				if err != nil {
					log.Printf("ERROR: failed parsing json: %s", err)
				}

				prettyPrint(m)
			}
		}
	}()

	wg.Wait()
	log.Println("Done consuming topic", cfg.Topic)
	close(messages)

	if err := c.Close(); err != nil {
		log.Fatal(fmt.Errorf("Failed to close consumer: %s", err))
	}
}