// handleGetTopicConsumers is an HTTP request handler for `GET /topic/{topic}/consumers`
func (as *T) handleGetTopicConsumers(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
var err error
topic := mux.Vars(r)[paramTopic]
group := ""
r.ParseForm()
groups := r.Form[paramGroup]
if len(groups) > 1 {
err = fmt.Errorf("One consumer group is expected, but %d provided", len(groups))
respondWithJSON(w, http.StatusBadRequest, errorHTTPResponse{err.Error()})
return
}
if len(groups) == 1 {
group = groups[0]
}
var consumers map[string]map[string][]int32
if group == "" {
consumers, err = as.admin.GetAllTopicConsumers(topic)
if err != nil {
respondWithJSON(w, http.StatusInternalServerError, errorHTTPResponse{err.Error()})
return
}
} else {
groupConsumers, err := as.admin.GetTopicConsumers(group, topic)
if err != nil {
if _, ok := err.(admin.ErrInvalidParam); ok {
respondWithJSON(w, http.StatusBadRequest, errorHTTPResponse{err.Error()})
return
}
respondWithJSON(w, http.StatusInternalServerError, errorHTTPResponse{err.Error()})
return
}
consumers = make(map[string]map[string][]int32)
if len(groupConsumers) != 0 {
consumers[group] = groupConsumers
}
}
encodedRes, err := json.MarshalIndent(consumers, "", " ")
if err != nil {
log.Errorf("Failed to send HTTP reponse: status=%d, body=%v, reason=%v", http.StatusOK, encodedRes, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
encodedRes = prettyfmt.CollapseJSON(encodedRes)
w.Header().Add(headerContentType, "application/json")
w.WriteHeader(http.StatusOK)
if _, err := w.Write(encodedRes); err != nil {
log.Errorf("Failed to send HTTP reponse: status=%d, body=%v, reason=%v", http.StatusOK, encodedRes, err)
}
}
// respondWithJSON marshals `body` to a JSON string and sends it as an HTTP
// response body along with the specified `status` code.
func respondWithJSON(w http.ResponseWriter, status int, body interface{}) {
encodedRes, err := json.MarshalIndent(body, "", " ")
if err != nil {
log.Errorf("Failed to send HTTP reponse: status=%d, body=%v, reason=%v", status, body, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Add(headerContentType, "application/json")
w.WriteHeader(status)
if _, err := w.Write(encodedRes); err != nil {
log.Errorf("Failed to send HTTP reponse: status=%d, body=%v, reason=%v", status, body, err)
}
}
// supervisor takes care of the service graceful shutdown.
func (s *Service) supervisor() {
defer sarama.RootCID.NewChild("supervisor").LogScope()()
var unixServerErrorCh <-chan error
s.tcpServer.Start()
if s.unixServer != nil {
s.unixServer.Start()
unixServerErrorCh = s.unixServer.ErrorCh()
}
// Block to wait for quit signal or an API server crash.
select {
case <-s.quitCh:
case err, ok := <-s.tcpServer.ErrorCh():
if ok {
log.Errorf("Unix socket based HTTP API crashed, err=(%s)", err)
}
case err, ok := <-unixServerErrorCh:
if ok {
log.Errorf("TCP socket based HTTP API crashed, err=(%s)", err)
}
}
// Initiate stop of all API servers.
s.tcpServer.AsyncStop()
if s.unixServer != nil {
s.unixServer.AsyncStop()
}
// Wait until all API servers are stopped.
for range s.tcpServer.ErrorCh() {
// Drain the errors channel until it is closed.
}
if s.unixServer != nil {
for range s.unixServer.ErrorCh() {
// Drain the errors channel until it is closed.
}
}
// There are no more requests in flight at this point so it is safe to stop
// all Kafka clients.
var wg sync.WaitGroup
spawn(&wg, s.producer.Stop)
spawn(&wg, s.consumer.Stop)
spawn(&wg, s.admin.Stop)
wg.Wait()
}
func main() {
// Make go runtime execute in parallel as many goroutines as there are CPUs.
runtime.GOMAXPROCS(runtime.NumCPU())
if err := initLogging(); err != nil {
fmt.Printf("Failed to initialize logger: err=(%s)\n", err)
os.Exit(1)
}
if pidFile != "" {
if err := writePID(pidFile); err != nil {
log.Errorf("Failed to write PID file: err=(%s)", err)
os.Exit(1)
}
}
// Clean up the unix domain socket file in case we failed to clean up on
// shutdown the last time. Otherwise the service won't be able to listen
// on this address and as a result will fail to start up.
if config.UnixAddr != "" {
if err := os.Remove(config.UnixAddr); err != nil && !os.IsNotExist(err) {
log.Errorf("Cannot remove %s: err=(%s)", config.UnixAddr, err)
}
}
log.Infof("Starting with config: %+v", config)
svc, err := pixy.SpawnService(config)
if err != nil {
log.Errorf("Failed to start service: err=(%s)", err)
os.Exit(1)
}
// Spawn OS signal listener to ensure graceful stop.
osSigCh := make(chan os.Signal, 1)
signal.Notify(osSigCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
// Wait for a quit signal and terminate the service when it is received.
<-osSigCh
svc.Stop()
}
// handleProduceResult inspects a production results and if it is an error
// then logs it and flushes it down the `deadMessageCh` if one had been
// configured.
func (gp *GracefulProducer) handleProduceResult(cid *sarama.ContextID, result produceResult) {
if replyCh, ok := result.Msg.Metadata.(chan produceResult); ok {
replyCh <- result
}
if result.Err == nil {
return
}
prodMsgRepr := fmt.Sprintf(`{Topic: "%s", Key: "%s", Value: "%s"}`,
result.Msg.Topic, encoderRepr(result.Msg.Key), encoderRepr(result.Msg.Value))
log.Errorf("<%v> Failed to submit message: msg=%v, err=(%s)",
cid, prodMsgRepr, result.Err)
if gp.deadMessageCh != nil {
gp.deadMessageCh <- result.Msg
}
}
// retry keeps calling the `f` function until it succeeds. `shouldRetry` is
// used to check the error code returned by `f` to decide whether it should be
// retried. If `shouldRetry` is not specified then any non `nil` error will
// result in retry.
func retry(f func() error, shouldRetry func(err error) bool, errorMsg string,
delay time.Duration, cancelCh <-chan none) (canceled bool) {
err := f()
if shouldRetry == nil {
shouldRetry = func(err error) bool { return err != nil }
}
for shouldRetry(err) {
log.Errorf("%s: err=(%s), retryIn=%v", errorMsg, err, delay)
select {
case <-time.After(delay):
case <-cancelCh:
return true
}
err = f()
}
return false
}
func (gc *groupConsumer) managePartitions() {
cid := gc.baseCID.NewChild("managePartitions")
defer cid.LogScope()()
var (
topicConsumers = make(map[string]*topicConsumer)
topics []string
memberSubscriptions map[string][]string
ok = true
nilOrRetryCh <-chan time.Time
nilOrRegistryTopicsCh chan<- []string
shouldRebalance, canRebalance = false, true
rebalanceResultCh = make(chan error, 1)
)
for {
select {
case tc := <-gc.addTopicConsumerCh:
topicConsumers[tc.topic] = tc
topics = listTopics(topicConsumers)
nilOrRegistryTopicsCh = gc.registry.topics()
continue
case tc := <-gc.deleteTopicConsumerCh:
delete(topicConsumers, tc.topic)
topics = listTopics(topicConsumers)
nilOrRegistryTopicsCh = gc.registry.topics()
continue
case nilOrRegistryTopicsCh <- topics:
nilOrRegistryTopicsCh = nil
continue
case memberSubscriptions, ok = <-gc.registry.membershipChanges():
if !ok {
goto done
}
nilOrRetryCh = nil
shouldRebalance = true
case err := <-rebalanceResultCh:
canRebalance = true
if err != nil {
log.Errorf("<%s> rebalance failed: err=(%s)", cid, err)
nilOrRetryCh = time.After(gc.config.Consumer.BackOffTimeout)
continue
}
case <-nilOrRetryCh:
shouldRebalance = true
}
if shouldRebalance && canRebalance {
// Copy topicConsumers to make sure `rebalance` doesn't see any
// changes we make while it is running.
topicConsumerCopy := make(map[string]*topicConsumer, len(topicConsumers))
for topic, tc := range topicConsumers {
topicConsumerCopy[topic] = tc
}
go gc.rebalance(topicConsumerCopy, memberSubscriptions, rebalanceResultCh)
shouldRebalance, canRebalance = false, false
}
}
done:
var wg sync.WaitGroup
for _, tg := range gc.topicGears {
tg := tg
spawn(&wg, func() { gc.rewireMultiplexer(tg, nil) })
}
wg.Wait()
}
func (gc *groupConsumer) managePartitions() {
cid := gc.baseCID.NewChild("managePartitions")
defer cid.LogScope()()
var (
topicConsumers = make(map[string]*topicConsumer)
topics []string
subscriptions map[string][]string
ok = true
nilOrRetryCh <-chan time.Time
nilOrRegistryTopicsCh chan<- []string
shouldRebalance, canRebalance = false, true
rebalanceResultCh = make(chan error, 1)
)
for {
select {
case tc := <-gc.topicConsumerLifespanCh:
// It is assumed that only one topicConsumer can exist for a
// particular topic at a time.
if topicConsumers[tc.topic] == tc {
delete(topicConsumers, tc.topic)
} else {
topicConsumers[tc.topic] = tc
}
topics = listTopics(topicConsumers)
nilOrRegistryTopicsCh = gc.registry.topics()
continue
case nilOrRegistryTopicsCh <- topics:
nilOrRegistryTopicsCh = nil
continue
case subscriptions, ok = <-gc.registry.membershipChanges():
if !ok {
goto done
}
nilOrRetryCh = nil
shouldRebalance = true
case err := <-rebalanceResultCh:
canRebalance = true
if err != nil {
log.Errorf("<%s> rebalance failed: err=(%s)", cid, err)
nilOrRetryCh = time.After(gc.cfg.Consumer.BackOffTimeout)
continue
}
case <-nilOrRetryCh:
shouldRebalance = true
}
if shouldRebalance && canRebalance {
// Copy topicConsumers to make sure `rebalance` doesn't see any
// changes we make while it is running.
topicConsumerCopy := make(map[string]*topicConsumer, len(topicConsumers))
for topic, tc := range topicConsumers {
topicConsumerCopy[topic] = tc
}
go gc.rebalance(topicConsumerCopy, subscriptions, rebalanceResultCh)
shouldRebalance, canRebalance = false, false
}
}
done:
var wg sync.WaitGroup
for _, tcg := range gc.topicConsumerGears {
spawn(&wg, tcg.stop)
}
wg.Wait()
}