Example #1
0
func GetTopics(kafkaHost string) []string {
	broker := sarama.NewBroker(kafkaHost)
	err := broker.Open(nil)
	if err != nil {
		panic(err)
	}

	request := sarama.MetadataRequest{ /*Topics: []string{"abba"}*/ }
	response, err := broker.GetMetadata(&request)
	if err != nil {
		_ = broker.Close()
		panic(err)
	}

	//topics := make([]string, len(response.Topics))
	topics := []string{}

	for _, v := range response.Topics {
		topics = append(topics, v.Name)
	}

	if err = broker.Close(); err != nil {
		panic(err)
	}

	return topics
}
Example #2
0
func mockNewBroker(t *testing.T, cp ChainPartition) (Broker, error) {
	mockBroker := sarama.NewMockBroker(t, testBrokerID)
	handlerMap := make(map[string]sarama.MockResponse)
	// The sarama mock package doesn't allow us to return an error
	// for invalid offset requests, so we return an offset of -1.
	// Note that the mock offset responses below imply a broker with
	// newestOffset-1 blocks available. Therefore, if you are using this
	// broker as part of a bigger test where you intend to consume blocks,
	// make sure that the mockConsumer has been initialized accordingly
	// (Set the 'offset' parameter to newestOffset-1.)
	handlerMap["OffsetRequest"] = sarama.NewMockOffsetResponse(t).
		SetOffset(cp.Topic(), cp.Partition(), sarama.OffsetOldest, testOldestOffset).
		SetOffset(cp.Topic(), cp.Partition(), sarama.OffsetNewest, testNewestOffset)
	mockBroker.SetHandlerByMap(handlerMap)

	broker := sarama.NewBroker(mockBroker.Addr())
	if err := broker.Open(nil); err != nil {
		return nil, fmt.Errorf("Cannot connect to mock broker: %s", err)
	}

	return &mockBrockerImpl{
		brokerImpl: brokerImpl{
			broker: broker,
		},
		mockBroker: mockBroker,
		handlerMap: handlerMap,
	}, nil
}
Example #3
0
func (s *TopicPartitionScreen) WillShow(screen Screen) {
	// get TopicPartition metadata
	broker := sarama.NewBroker(s.broker)

	config := sarama.NewConfig()

	err := broker.Open(config)
	if err != nil {
		panic(err)
	}
	defer broker.Close()

	// first, obtain meta data about the topic
	var topics [1]string
	topics[0] = s.topic

	topicMetaReq := &sarama.MetadataRequest{
		Topics: topics[:],
	}

	metadata, err := broker.GetMetadata(topicMetaReq)
	if err != nil {
		log.Println(err)
		// panic(err)
	}

	s.brokers = metadata.Brokers
	s.topics = metadata.Topics
	s.partitions = s.topics[0].Partitions
	sort.Sort(s.partitions)
}
Example #4
0
// Connects to the broker that handles all produce and consume
// requests for the given chain (Partition Leader Replica)
func newBroker(brokers []string, cp ChainPartition) (Broker, error) {
	var candidateBroker, connectedBroker, leaderBroker *sarama.Broker

	// Connect to one of the given brokers
	for _, hostPort := range brokers {
		candidateBroker = sarama.NewBroker(hostPort)
		if err := candidateBroker.Open(nil); err != nil {
			logger.Warningf("Failed to connect to broker %s: %s", hostPort, err)
			continue
		}
		if connected, err := candidateBroker.Connected(); !connected {
			logger.Warningf("Failed to connect to broker %s: %s", hostPort, err)
			continue
		}
		connectedBroker = candidateBroker
		break
	}

	if connectedBroker == nil {
		return nil, fmt.Errorf("Failed to connect to any of the given brokers (%v) for metadata request", brokers)
	}
	logger.Debugf("Connected to broker %s", connectedBroker.Addr())

	// Get metadata for the topic that corresponds to this chain
	metadata, err := connectedBroker.GetMetadata(&sarama.MetadataRequest{Topics: []string{cp.Topic()}})
	if err != nil {
		return nil, fmt.Errorf("Failed to get metadata for topic %s: %s", cp, err)
	}

	// Get the leader broker for this chain partition
	if (cp.Partition() >= 0) && (cp.Partition() < int32(len(metadata.Topics[0].Partitions))) {
		leaderBrokerID := metadata.Topics[0].Partitions[cp.Partition()].Leader
		logger.Debugf("Leading broker for chain %s is broker ID %d", cp, leaderBrokerID)
		for _, availableBroker := range metadata.Brokers {
			if availableBroker.ID() == leaderBrokerID {
				leaderBroker = availableBroker
				break
			}
		}
	}

	if leaderBroker == nil {
		return nil, fmt.Errorf("Can't find leader for chain %s", cp)
	}

	// Connect to broker
	if err := leaderBroker.Open(nil); err != nil {
		return nil, fmt.Errorf("Failed to connect ho Kafka broker: %s", err)
	}
	if connected, err := leaderBroker.Connected(); !connected {
		return nil, fmt.Errorf("Failed to connect to Kafka broker: %s", err)
	}

	return &brokerImpl{broker: leaderBroker}, nil
}
Example #5
0
func (b *Broker) Metadata(req *sarama.MetadataRequest) *sarama.MetadataResponse {
	response := &sarama.MetadataResponse{}
	response.Brokers = []*sarama.Broker{sarama.NewBroker(b.addr)}
	if len(req.Topics) == 0 {
		for topic := range b.partitions {
			response.Topics = append(response.Topics, b.metadataForTopic(topic))
		}
	} else {
		for _, topic := range req.Topics {
			response.Topics = append(response.Topics, b.metadataForTopic(topic))
		}
	}
	return response
}
Example #6
0
func main() {
	broker := kafka.NewBroker("localhost:9092")
	err := broker.Open(nil)
	if err != nil {
		panic(err)
	}
	defer broker.Close()

	request := kafka.MetadataRequest{Topics: []string{"myTopic"}}
	response, err := broker.GetMetadata("myClient", &request)
	if err != nil {
		panic(err)
	}

	fmt.Println("There are", len(response.Topics), "topics active in the cluster.")
}
Example #7
0
// this happens aynchronously, so calling GetTopics() immediately afterwards
// will likely not show you your new topic
func AddTopic(kafkaHost string, topic string) {
	broker := sarama.NewBroker(kafkaHost)
	err := broker.Open(nil)
	if err != nil {
		panic(err)
	}

	request := sarama.MetadataRequest{Topics: []string{topic}}
	_, err = broker.GetMetadata(&request)
	if err != nil {
		_ = broker.Close()
		panic(err)
	}

	if err = broker.Close(); err != nil {
		panic(err)
	}
}
Example #8
0
func main() {
	addr := os.Args[1]
	fmt.Println("try to connect to,", addr)

	broker := sarama.NewBroker(addr)
	err := broker.Open(nil)
	if err != nil {
		fmt.Println("failed to open broker: ", err)
		os.Exit(1)
	}

	defer broker.Close()

	var topics []string
	if len(os.Args) > 2 {
		topics = os.Args[2:]
		fmt.Println("query for topics: ", topics)
	}

	request := sarama.MetadataRequest{Topics: topics}
	response, err := broker.GetMetadata(&request)
	if err != nil {
		fmt.Println("metadata request failed with: ", err)
		os.Exit(1)
	}

	fmt.Println("known topics")
	for _, t := range response.Topics {
		fmt.Println("topic")
		fmt.Println("  name: ", t.Name)
		fmt.Println("  error: ", t.Err)
		fmt.Println("  partitions: ")
		for _, p := range t.Partitions {
			fmt.Println("    ", p.ID)
		}
	}

	fmt.Println("broker addresses")
	for _, b := range response.Brokers {
		fmt.Println(b.Addr())
	}
}
Example #9
0
func (b *Broker) getData(partition int32, offset int64) ([]byte, error) {
	// TODO: using pool ?
	bb, err := b.bc.Leader(gBrokerTopic, partition)
	if err != nil {
		return nil, err
	}
	bbb := sarama.NewBroker(bb.Addr())
	defer bbb.Close()
	err = bbb.Open(b.brokerConfig)
	if err != nil {
		return nil, err
	}
	freq := &sarama.FetchRequest{}
	freq.AddBlock(gBrokerTopic, partition, offset, int32(b.config.brokerMaxMessageSize))
	fres, err := bbb.Fetch(freq)
	if err != nil {
		return nil, err
	}
	fresb := fres.GetBlock(gBrokerTopic, partition)
	msgs := fresb.MsgSet.Messages
	if len(msgs) == 0 {
		return nil, errors.New("no msg found in broker")
	}
	/* unpack */
	buf := bytes.NewReader(msgs[0].Msg.Value)
	buf.Seek(0, 0)
	msgr := msgp.NewReader(buf)
	resi, err := msgr.ReadIntf()
	if err != nil {
		return nil, errors.New(fmt.Sprintf("fail to decode data from broker: %s", err.Error()))
	}
	res, ok := resi.(map[string]interface{})
	if !ok {
		return nil, errors.New(fmt.Sprintf("invalid data from broker: data need be map"))
	}
	datai, ok := res["data"]
	if !ok {
		return nil, errors.New(fmt.Sprintf("invalid data from broker: data not exist"))
	}
	return datai.([]byte), nil
}
Example #10
0
File: main.go Project: yichen/ktop
func main() {
	var seedBrokerStr string
	args := os.Args
	if len(args) == 1 {
		seedBrokerStr = "eat1-app1252.corp.linkedin.com:10251"
	} else {
		seedBrokerStr = args[1]
	}

	seedBroker := sarama.NewBroker(seedBrokerStr)
	config := sarama.NewConfig()

	err := seedBroker.Open(config)
	if err != nil {
		panic(err)
	}
	defer seedBroker.Close()

	offsetRequest := &sarama.OffsetRequest{}

	for i := 0; i < 8; i++ {
		offsetRequest.AddBlock("TestDBForKET", int32(i), sarama.LatestOffsets, 1)
	}

	offsetResponse, err := seedBroker.GetAvailableOffsets("ktop", offsetRequest)
	if err != nil {
		panic(err)
	}

	for i := 0; i < 8; i++ {
		offsetResponseBlock := offsetResponse.GetBlock("TestDBForKET", int32(i))
		for _, offset := range offsetResponseBlock.Offsets {
			fmt.Println(offset)
		}
		fmt.Println("--")
	}

}
Example #11
0
// New create a new instance of the partition MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
	config := defaultConfig
	if err := base.Module().UnpackConfig(&config); err != nil {
		return nil, err
	}

	tls, err := outputs.LoadTLSConfig(config.TLS)
	if err != nil {
		return nil, err
	}

	cfg := sarama.NewConfig()
	cfg.Net.DialTimeout = base.Module().Config().Timeout
	cfg.Net.ReadTimeout = base.Module().Config().Timeout
	cfg.ClientID = config.ClientID
	cfg.Metadata.Retry.Max = config.Metadata.Retries
	cfg.Metadata.Retry.Backoff = config.Metadata.Backoff
	if tls != nil {
		cfg.Net.TLS.Enable = true
		cfg.Net.TLS.Config = tls.BuildModuleConfig("")
	}
	if config.Username != "" {
		cfg.Net.SASL.Enable = true
		cfg.Net.SASL.User = config.Username
		cfg.Net.SASL.Password = config.Password
	}

	broker := sarama.NewBroker(base.Host())
	return &MetricSet{
		BaseMetricSet: base,
		broker:        broker,
		cfg:           cfg,
		id:            noID,
		topics:        config.Topics,
	}, nil
}
Example #12
0
func mustFindLeaders() map[int32]*sarama.Broker {
	topic := config.produce.topic
	conf := sarama.NewConfig()
	conf.Producer.RequiredAcks = sarama.WaitForAll
	conf.Version = config.produce.version
	u, err := user.Current()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read current user err=%v", err)
	}
	conf.ClientID = "kt-produce-" + u.Username
	if config.produce.verbose {
		fmt.Fprintf(os.Stderr, "sarama client configuration %#v\n", conf)
	}
	metaReq := sarama.MetadataRequest{Topics: []string{topic}}

tryingBrokers:
	for _, brokerString := range config.produce.brokers {
		broker := sarama.NewBroker(brokerString)
		err := broker.Open(conf)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to open broker connection to %v. err=%s\n", brokerString, err)
			continue tryingBrokers
		}
		if connected, err := broker.Connected(); !connected || err != nil {
			fmt.Fprintf(os.Stderr, "Failed to open broker connection to %v. err=%s\n", brokerString, err)
			continue tryingBrokers
		}

		metaResp, err := broker.GetMetadata(&metaReq)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to get metadata from [%v]. err=%v\n", brokerString, err)
			continue tryingBrokers
		}

		brokers := map[int32]*sarama.Broker{}
		for _, b := range metaResp.Brokers {
			brokers[b.ID()] = b
		}

		for _, tm := range metaResp.Topics {
			if tm.Name == topic {
				if tm.Err != sarama.ErrNoError {
					fmt.Fprintf(os.Stderr, "Failed to get metadata from %v. err=%v\n", brokerString, tm.Err)
					continue tryingBrokers
				}

				bs := map[int32]*sarama.Broker{}
				for _, pm := range tm.Partitions {
					b, ok := brokers[pm.Leader]
					if !ok {
						fmt.Fprintf(os.Stderr, "Failed to find leader in broker response, giving up.\n")
						os.Exit(1)
					}

					err := b.Open(conf)
					if err != nil && err != sarama.ErrAlreadyConnected {
						fmt.Fprintf(os.Stderr, "Failed to open broker connection. err=%s\n", err)
						os.Exit(1)
					}
					if connected, err := broker.Connected(); !connected && err != nil {
						fmt.Fprintf(os.Stderr, "Failed to wait for broker connection to open. err=%s\n", err)
						os.Exit(1)
					}

					bs[pm.ID] = b
				}
				return bs
			}
		}
	}

	fmt.Fprintf(os.Stderr, "Failed to find leader for given topic.\n")
	os.Exit(1)
	return nil
}