func GetTopics(kafkaHost string) []string {
broker := sarama.NewBroker(kafkaHost)
err := broker.Open(nil)
if err != nil {
panic(err)
}
request := sarama.MetadataRequest{ /*Topics: []string{"abba"}*/ }
response, err := broker.GetMetadata(&request)
if err != nil {
_ = broker.Close()
panic(err)
}
//topics := make([]string, len(response.Topics))
topics := []string{}
for _, v := range response.Topics {
topics = append(topics, v.Name)
}
if err = broker.Close(); err != nil {
panic(err)
}
return topics
}
func mockNewBroker(t *testing.T, cp ChainPartition) (Broker, error) {
mockBroker := sarama.NewMockBroker(t, testBrokerID)
handlerMap := make(map[string]sarama.MockResponse)
// The sarama mock package doesn't allow us to return an error
// for invalid offset requests, so we return an offset of -1.
// Note that the mock offset responses below imply a broker with
// newestOffset-1 blocks available. Therefore, if you are using this
// broker as part of a bigger test where you intend to consume blocks,
// make sure that the mockConsumer has been initialized accordingly
// (Set the 'offset' parameter to newestOffset-1.)
handlerMap["OffsetRequest"] = sarama.NewMockOffsetResponse(t).
SetOffset(cp.Topic(), cp.Partition(), sarama.OffsetOldest, testOldestOffset).
SetOffset(cp.Topic(), cp.Partition(), sarama.OffsetNewest, testNewestOffset)
mockBroker.SetHandlerByMap(handlerMap)
broker := sarama.NewBroker(mockBroker.Addr())
if err := broker.Open(nil); err != nil {
return nil, fmt.Errorf("Cannot connect to mock broker: %s", err)
}
return &mockBrockerImpl{
brokerImpl: brokerImpl{
broker: broker,
},
mockBroker: mockBroker,
handlerMap: handlerMap,
}, nil
}
func (s *TopicPartitionScreen) WillShow(screen Screen) {
// get TopicPartition metadata
broker := sarama.NewBroker(s.broker)
config := sarama.NewConfig()
err := broker.Open(config)
if err != nil {
panic(err)
}
defer broker.Close()
// first, obtain meta data about the topic
var topics [1]string
topics[0] = s.topic
topicMetaReq := &sarama.MetadataRequest{
Topics: topics[:],
}
metadata, err := broker.GetMetadata(topicMetaReq)
if err != nil {
log.Println(err)
// panic(err)
}
s.brokers = metadata.Brokers
s.topics = metadata.Topics
s.partitions = s.topics[0].Partitions
sort.Sort(s.partitions)
}
// Connects to the broker that handles all produce and consume
// requests for the given chain (Partition Leader Replica)
func newBroker(brokers []string, cp ChainPartition) (Broker, error) {
var candidateBroker, connectedBroker, leaderBroker *sarama.Broker
// Connect to one of the given brokers
for _, hostPort := range brokers {
candidateBroker = sarama.NewBroker(hostPort)
if err := candidateBroker.Open(nil); err != nil {
logger.Warningf("Failed to connect to broker %s: %s", hostPort, err)
continue
}
if connected, err := candidateBroker.Connected(); !connected {
logger.Warningf("Failed to connect to broker %s: %s", hostPort, err)
continue
}
connectedBroker = candidateBroker
break
}
if connectedBroker == nil {
return nil, fmt.Errorf("Failed to connect to any of the given brokers (%v) for metadata request", brokers)
}
logger.Debugf("Connected to broker %s", connectedBroker.Addr())
// Get metadata for the topic that corresponds to this chain
metadata, err := connectedBroker.GetMetadata(&sarama.MetadataRequest{Topics: []string{cp.Topic()}})
if err != nil {
return nil, fmt.Errorf("Failed to get metadata for topic %s: %s", cp, err)
}
// Get the leader broker for this chain partition
if (cp.Partition() >= 0) && (cp.Partition() < int32(len(metadata.Topics[0].Partitions))) {
leaderBrokerID := metadata.Topics[0].Partitions[cp.Partition()].Leader
logger.Debugf("Leading broker for chain %s is broker ID %d", cp, leaderBrokerID)
for _, availableBroker := range metadata.Brokers {
if availableBroker.ID() == leaderBrokerID {
leaderBroker = availableBroker
break
}
}
}
if leaderBroker == nil {
return nil, fmt.Errorf("Can't find leader for chain %s", cp)
}
// Connect to broker
if err := leaderBroker.Open(nil); err != nil {
return nil, fmt.Errorf("Failed to connect ho Kafka broker: %s", err)
}
if connected, err := leaderBroker.Connected(); !connected {
return nil, fmt.Errorf("Failed to connect to Kafka broker: %s", err)
}
return &brokerImpl{broker: leaderBroker}, nil
}
func (b *Broker) Metadata(req *sarama.MetadataRequest) *sarama.MetadataResponse {
response := &sarama.MetadataResponse{}
response.Brokers = []*sarama.Broker{sarama.NewBroker(b.addr)}
if len(req.Topics) == 0 {
for topic := range b.partitions {
response.Topics = append(response.Topics, b.metadataForTopic(topic))
}
} else {
for _, topic := range req.Topics {
response.Topics = append(response.Topics, b.metadataForTopic(topic))
}
}
return response
}
func main() {
broker := kafka.NewBroker("localhost:9092")
err := broker.Open(nil)
if err != nil {
panic(err)
}
defer broker.Close()
request := kafka.MetadataRequest{Topics: []string{"myTopic"}}
response, err := broker.GetMetadata("myClient", &request)
if err != nil {
panic(err)
}
fmt.Println("There are", len(response.Topics), "topics active in the cluster.")
}
// this happens aynchronously, so calling GetTopics() immediately afterwards
// will likely not show you your new topic
func AddTopic(kafkaHost string, topic string) {
broker := sarama.NewBroker(kafkaHost)
err := broker.Open(nil)
if err != nil {
panic(err)
}
request := sarama.MetadataRequest{Topics: []string{topic}}
_, err = broker.GetMetadata(&request)
if err != nil {
_ = broker.Close()
panic(err)
}
if err = broker.Close(); err != nil {
panic(err)
}
}
func main() {
addr := os.Args[1]
fmt.Println("try to connect to,", addr)
broker := sarama.NewBroker(addr)
err := broker.Open(nil)
if err != nil {
fmt.Println("failed to open broker: ", err)
os.Exit(1)
}
defer broker.Close()
var topics []string
if len(os.Args) > 2 {
topics = os.Args[2:]
fmt.Println("query for topics: ", topics)
}
request := sarama.MetadataRequest{Topics: topics}
response, err := broker.GetMetadata(&request)
if err != nil {
fmt.Println("metadata request failed with: ", err)
os.Exit(1)
}
fmt.Println("known topics")
for _, t := range response.Topics {
fmt.Println("topic")
fmt.Println(" name: ", t.Name)
fmt.Println(" error: ", t.Err)
fmt.Println(" partitions: ")
for _, p := range t.Partitions {
fmt.Println(" ", p.ID)
}
}
fmt.Println("broker addresses")
for _, b := range response.Brokers {
fmt.Println(b.Addr())
}
}
func (b *Broker) getData(partition int32, offset int64) ([]byte, error) {
// TODO: using pool ?
bb, err := b.bc.Leader(gBrokerTopic, partition)
if err != nil {
return nil, err
}
bbb := sarama.NewBroker(bb.Addr())
defer bbb.Close()
err = bbb.Open(b.brokerConfig)
if err != nil {
return nil, err
}
freq := &sarama.FetchRequest{}
freq.AddBlock(gBrokerTopic, partition, offset, int32(b.config.brokerMaxMessageSize))
fres, err := bbb.Fetch(freq)
if err != nil {
return nil, err
}
fresb := fres.GetBlock(gBrokerTopic, partition)
msgs := fresb.MsgSet.Messages
if len(msgs) == 0 {
return nil, errors.New("no msg found in broker")
}
/* unpack */
buf := bytes.NewReader(msgs[0].Msg.Value)
buf.Seek(0, 0)
msgr := msgp.NewReader(buf)
resi, err := msgr.ReadIntf()
if err != nil {
return nil, errors.New(fmt.Sprintf("fail to decode data from broker: %s", err.Error()))
}
res, ok := resi.(map[string]interface{})
if !ok {
return nil, errors.New(fmt.Sprintf("invalid data from broker: data need be map"))
}
datai, ok := res["data"]
if !ok {
return nil, errors.New(fmt.Sprintf("invalid data from broker: data not exist"))
}
return datai.([]byte), nil
}
func main() {
var seedBrokerStr string
args := os.Args
if len(args) == 1 {
seedBrokerStr = "eat1-app1252.corp.linkedin.com:10251"
} else {
seedBrokerStr = args[1]
}
seedBroker := sarama.NewBroker(seedBrokerStr)
config := sarama.NewConfig()
err := seedBroker.Open(config)
if err != nil {
panic(err)
}
defer seedBroker.Close()
offsetRequest := &sarama.OffsetRequest{}
for i := 0; i < 8; i++ {
offsetRequest.AddBlock("TestDBForKET", int32(i), sarama.LatestOffsets, 1)
}
offsetResponse, err := seedBroker.GetAvailableOffsets("ktop", offsetRequest)
if err != nil {
panic(err)
}
for i := 0; i < 8; i++ {
offsetResponseBlock := offsetResponse.GetBlock("TestDBForKET", int32(i))
for _, offset := range offsetResponseBlock.Offsets {
fmt.Println(offset)
}
fmt.Println("--")
}
}
// New create a new instance of the partition MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
config := defaultConfig
if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
cfg := sarama.NewConfig()
cfg.Net.DialTimeout = base.Module().Config().Timeout
cfg.Net.ReadTimeout = base.Module().Config().Timeout
cfg.ClientID = config.ClientID
cfg.Metadata.Retry.Max = config.Metadata.Retries
cfg.Metadata.Retry.Backoff = config.Metadata.Backoff
if tls != nil {
cfg.Net.TLS.Enable = true
cfg.Net.TLS.Config = tls.BuildModuleConfig("")
}
if config.Username != "" {
cfg.Net.SASL.Enable = true
cfg.Net.SASL.User = config.Username
cfg.Net.SASL.Password = config.Password
}
broker := sarama.NewBroker(base.Host())
return &MetricSet{
BaseMetricSet: base,
broker: broker,
cfg: cfg,
id: noID,
topics: config.Topics,
}, nil
}
func mustFindLeaders() map[int32]*sarama.Broker {
topic := config.produce.topic
conf := sarama.NewConfig()
conf.Producer.RequiredAcks = sarama.WaitForAll
conf.Version = config.produce.version
u, err := user.Current()
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to read current user err=%v", err)
}
conf.ClientID = "kt-produce-" + u.Username
if config.produce.verbose {
fmt.Fprintf(os.Stderr, "sarama client configuration %#v\n", conf)
}
metaReq := sarama.MetadataRequest{Topics: []string{topic}}
tryingBrokers:
for _, brokerString := range config.produce.brokers {
broker := sarama.NewBroker(brokerString)
err := broker.Open(conf)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to open broker connection to %v. err=%s\n", brokerString, err)
continue tryingBrokers
}
if connected, err := broker.Connected(); !connected || err != nil {
fmt.Fprintf(os.Stderr, "Failed to open broker connection to %v. err=%s\n", brokerString, err)
continue tryingBrokers
}
metaResp, err := broker.GetMetadata(&metaReq)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to get metadata from [%v]. err=%v\n", brokerString, err)
continue tryingBrokers
}
brokers := map[int32]*sarama.Broker{}
for _, b := range metaResp.Brokers {
brokers[b.ID()] = b
}
for _, tm := range metaResp.Topics {
if tm.Name == topic {
if tm.Err != sarama.ErrNoError {
fmt.Fprintf(os.Stderr, "Failed to get metadata from %v. err=%v\n", brokerString, tm.Err)
continue tryingBrokers
}
bs := map[int32]*sarama.Broker{}
for _, pm := range tm.Partitions {
b, ok := brokers[pm.Leader]
if !ok {
fmt.Fprintf(os.Stderr, "Failed to find leader in broker response, giving up.\n")
os.Exit(1)
}
err := b.Open(conf)
if err != nil && err != sarama.ErrAlreadyConnected {
fmt.Fprintf(os.Stderr, "Failed to open broker connection. err=%s\n", err)
os.Exit(1)
}
if connected, err := broker.Connected(); !connected && err != nil {
fmt.Fprintf(os.Stderr, "Failed to wait for broker connection to open. err=%s\n", err)
os.Exit(1)
}
bs[pm.ID] = b
}
return bs
}
}
}
fmt.Fprintf(os.Stderr, "Failed to find leader for given topic.\n")
os.Exit(1)
return nil
}