コード例 #1
0
func parseAndValidateArgs() *kafka.MirrorMakerConfig {
	flag.Var(&consumerConfig, "consumer.config", "Path to consumer configuration file.")
	flag.Parse()
	runtime.GOMAXPROCS(*maxProcs)

	if (*whitelist != "" && *blacklist != "") || (*whitelist == "" && *blacklist == "") {
		fmt.Println("Exactly one of whitelist or blacklist is required.")
		os.Exit(1)
	}
	if *producerConfig == "" {
		fmt.Println("Producer config is required.")
		os.Exit(1)
	}
	if len(consumerConfig) == 0 {
		fmt.Println("At least one consumer config is required.")
		os.Exit(1)
	}
	if *queueSize < 0 {
		fmt.Println("Queue size should be equal or greater than 0")
		os.Exit(1)
	}

	config := kafka.NewMirrorMakerConfig()
	config.Blacklist = *blacklist
	config.Whitelist = *whitelist
	config.ChannelSize = *queueSize
	config.ConsumerConfigs = []string(consumerConfig)
	config.NumProducers = *numProducers
	config.NumStreams = *numStreams
	config.PreservePartitions = *preservePartitions
	config.PreserveOrder = *preserveOrder
	config.ProducerConfig = *producerConfig
	config.TopicPrefix = *prefix
	if *schemaRegistryUrl != "" {
		config.KeyEncoder = avro.NewKafkaAvroEncoder(*schemaRegistryUrl).Encode
		config.ValueEncoder = avro.NewKafkaAvroEncoder(*schemaRegistryUrl).Encode
		config.KeyDecoder = avro.NewKafkaAvroDecoder(*schemaRegistryUrl)
		config.ValueDecoder = avro.NewKafkaAvroDecoder(*schemaRegistryUrl)
	}

	return config
}
コード例 #2
0
func (e *MirrorMakerExecutor) startMirrorMaker() {
	consumerConfigs := make([]string, 0)
	rawConsumerConfigs := strings.Split(e.config["consumer.config"], ",")
	for _, config := range rawConsumerConfigs {
		resourceTokens := strings.Split(config, "/")
		consumerConfigs = append(consumerConfigs, resourceTokens[len(resourceTokens)-1])
	}

	rawProducerConfig := strings.Split(e.config["producer.config"], "/")
	producerConfig := rawProducerConfig[len(rawProducerConfig)-1]

	mmConfig := kafka.NewMirrorMakerConfig()
	e.config.SetIntConfig("num.producers", &mmConfig.NumProducers)
	e.config.SetIntConfig("num.streams", &mmConfig.NumStreams)
	e.config.SetStringConfig("prefix", &mmConfig.TopicPrefix)
	e.config.SetIntConfig("queue.size", &mmConfig.ChannelSize)
	e.config.SetStringConfig("whitelist", &mmConfig.Whitelist)
	e.config.SetStringConfig("blacklist", &mmConfig.Blacklist)
	mmConfig.ProducerConfig = producerConfig
	mmConfig.ConsumerConfigs = consumerConfigs

	e.mirrorMaker = kafka.NewMirrorMaker(mmConfig)
	e.mirrorMaker.Start()
}