Esempio n. 1
0
// Start a rebalance cycle
func (cg *ConsumerGroup) rebalance() (err error) {
	var cids []string
	var pids []int32

	// Fetch a list of consumers and listen for changes
	if cids, cg.zkchange, err = cg.zoo.Consumers(cg.name); err != nil {
		cg.zkchange = nil
		return
	}

	// Fetch a list of partition IDs
	if pids, err = cg.client.Partitions(cg.topic); err != nil {
		cg.zkchange = nil
		return
	}

	// Get leaders for each partition ID
	parts := make(PartitionSlice, len(pids))
	for i, pid := range pids {
		var broker *sarama.Broker
		if broker, err = cg.client.Leader(cg.topic, pid); err != nil {
			cg.zkchange = nil
			return
		}
		defer broker.Close()
		parts[i] = Partition{Id: pid, Addr: broker.Addr()}
	}

	if err = cg.makeClaims(cids, parts); err != nil {
		cg.zkchange = nil
		cg.releaseClaims()
		return
	}
	return
}
Esempio n. 2
0
// Connects to the broker that handles all produce and consume
// requests for the given chain (Partition Leader Replica)
func newBroker(brokers []string, cp ChainPartition) (Broker, error) {
	var candidateBroker, connectedBroker, leaderBroker *sarama.Broker

	// Connect to one of the given brokers
	for _, hostPort := range brokers {
		candidateBroker = sarama.NewBroker(hostPort)
		if err := candidateBroker.Open(nil); err != nil {
			logger.Warningf("Failed to connect to broker %s: %s", hostPort, err)
			continue
		}
		if connected, err := candidateBroker.Connected(); !connected {
			logger.Warningf("Failed to connect to broker %s: %s", hostPort, err)
			continue
		}
		connectedBroker = candidateBroker
		break
	}

	if connectedBroker == nil {
		return nil, fmt.Errorf("Failed to connect to any of the given brokers (%v) for metadata request", brokers)
	}
	logger.Debugf("Connected to broker %s", connectedBroker.Addr())

	// Get metadata for the topic that corresponds to this chain
	metadata, err := connectedBroker.GetMetadata(&sarama.MetadataRequest{Topics: []string{cp.Topic()}})
	if err != nil {
		return nil, fmt.Errorf("Failed to get metadata for topic %s: %s", cp, err)
	}

	// Get the leader broker for this chain partition
	if (cp.Partition() >= 0) && (cp.Partition() < int32(len(metadata.Topics[0].Partitions))) {
		leaderBrokerID := metadata.Topics[0].Partitions[cp.Partition()].Leader
		logger.Debugf("Leading broker for chain %s is broker ID %d", cp, leaderBrokerID)
		for _, availableBroker := range metadata.Brokers {
			if availableBroker.ID() == leaderBrokerID {
				leaderBroker = availableBroker
				break
			}
		}
	}

	if leaderBroker == nil {
		return nil, fmt.Errorf("Can't find leader for chain %s", cp)
	}

	// Connect to broker
	if err := leaderBroker.Open(nil); err != nil {
		return nil, fmt.Errorf("Failed to connect ho Kafka broker: %s", err)
	}
	if connected, err := leaderBroker.Connected(); !connected {
		return nil, fmt.Errorf("Failed to connect to Kafka broker: %s", err)
	}

	return &brokerImpl{broker: leaderBroker}, nil
}
Esempio n. 3
0
func brokerAddress(b *sarama.Broker) string {
	return strings.ToLower(b.Addr())
}