func NewElodinaTransportScheduler(config ElodinaTransportSchedulerConfig) *ElodinaTransportScheduler {
connectorConfig := siesta.NewConnectorConfig()
connectorConfig.BrokerList = config.ConsumerConfig.BrokerList
connectorConfig.ClientID = config.ConsumerConfig.ClientID
connectorConfig.CommitOffsetBackoff = config.ConsumerConfig.CommitOffsetBackoff
connectorConfig.CommitOffsetRetries = config.ConsumerConfig.CommitOffsetRetries
connectorConfig.ConnectTimeout = config.ConsumerConfig.ConnectTimeout
connectorConfig.ConsumerMetadataBackoff = config.ConsumerConfig.ConsumerMetadataBackoff
connectorConfig.ConsumerMetadataRetries = config.ConsumerConfig.ConsumerMetadataRetries
connectorConfig.FetchMaxWaitTime = config.ConsumerConfig.FetchMaxWaitTime
connectorConfig.FetchMinBytes = config.ConsumerConfig.FetchMinBytes
connectorConfig.FetchSize = config.ConsumerConfig.FetchSize
connectorConfig.KeepAlive = config.ConsumerConfig.KeepAlive
connectorConfig.KeepAliveTimeout = config.ConsumerConfig.KeepAliveTimeout
connectorConfig.MaxConnections = config.ConsumerConfig.MaxConnections
connectorConfig.MaxConnectionsPerBroker = config.ConsumerConfig.MaxConnectionsPerBroker
connectorConfig.MetadataBackoff = config.ConsumerConfig.MetadataBackoff
connectorConfig.MetadataRetries = config.ConsumerConfig.MetadataRetries
connectorConfig.ReadTimeout = config.ConsumerConfig.ReadTimeout
connectorConfig.WriteTimeout = config.ConsumerConfig.WriteTimeout
kafkaClient, err := siesta.NewDefaultConnector(connectorConfig)
if err != nil {
panic(err)
}
scheduler := &ElodinaTransportScheduler{
config: &config,
taskIdToTaskState: make(map[string]*ElodinaTransport),
kafkaClient: kafkaClient,
}
scheduler.TakenTopicPartitions = consumer.NewTopicAndPartitionSet()
return scheduler
}
func (this *ElodinaTransportScheduler) GetTopicPartitions() (*consumer.TopicAndPartitionSet, error) {
topicsMetadata, err := this.kafkaClient.GetTopicMetadata(this.config.Topics)
if err != nil {
return nil, err
}
topicsAndPartitions := make([]consumer.TopicAndPartition, 0)
for _, topicMetadata := range topicsMetadata.TopicsMetadata {
for _, partitionMetadata := range topicMetadata.PartitionsMetadata {
topicsAndPartitions = append(topicsAndPartitions, consumer.TopicAndPartition{
Topic: topicMetadata.Topic,
Partition: partitionMetadata.PartitionID,
})
}
}
tpSet := consumer.NewTopicAndPartitionSet()
tpSet.AddAll(topicsAndPartitions)
log.Logger.Debug("%v", topicsAndPartitions)
log.Logger.Debug("%v", tpSet.GetArray())
return tpSet, nil
}