func NewKafkaInst(w *logrus.Logger, conf kafka_config) *Kafka { var err error var offset int64 config := sarama.NewConfig() // init partitioner switch conf.Partitioner { case "hash": config.Producer.Partitioner = sarama.NewHashPartitioner case "random": config.Producer.Partitioner = sarama.NewRandomPartitioner case "manual": config.Producer.Partitioner = sarama.NewManualPartitioner if conf.Partition == -1 { w.WithFields(logrus.Fields{ "module": "kafka", }).Fatalln("Partition is required when partitioning manually.") } default: w.WithFields(logrus.Fields{ "module": "kafka", }).Fatalf("Partitioner %s not supported.\n", conf.Partitioner) } if conf.Read_from_oldest { offset = sarama.OffsetOldest } else { offset = sarama.OffsetNewest } partition := int32(conf.Partition) // init topic topic := conf.Topics config.Producer.RequiredAcks = sarama.WaitForAll // init producer brokerlist := conf.Brokers producer, err := sarama.NewSyncProducer(brokerlist, config) if err != nil { w.WithFields(logrus.Fields{ "module": "kafka", }).Fatalln("Init failed:", err) } w.WithFields(logrus.Fields{ "module": "kafka", }).Infoln("Init completed") return &Kafka{producer: producer, topic: topic, partition: partition, brokerlist: brokerlist, logger: w, offset: offset} }
// Init initializes a avro package func NewAvroInst(w *logrus.Logger, conf avro_config) *Avro { data, err := ioutil.ReadFile(conf.Schema) if err != nil { w.WithFields(logrus.Fields{ "module": "avro", }).Fatalln("Failed to open schema file:", err) } recordSchemaJSON := string(data) codec, err := goavro.NewCodec(recordSchemaJSON) if err != nil { w.WithFields(logrus.Fields{ "module": "avro", }).Fatalln("Failed to init codec from schema:", err) } w.WithFields(logrus.Fields{ "module": "avro", }).Println("Init completed.") return &Avro{codec: codec, recordSchemaJSON: recordSchemaJSON, logger: w} }