func (*Sample) produceSample() string {
return fmt.Sprintf(`
public class KafkaProducer {
private final Producer<String, String> producer;
public final static String TOPIC = "TEST-TOPIC";
private KafkaProducer(){
Properties props = new Properties();
props.put("%s", "k10101a.mycorp.kfk.com:10101,k10101b.mycorp.kfk.com:10101");
props.put("producer.type", "sync"); // sync | async
props.put("serializer.class", "kafka.serializer.StringEncoder");
props.put("key.serializer.class", "kafka.serializer.StringEncoder");
// batch related
props.put("batch.num.messages", 200); // by default 200
props.put("queue.buffering.max.ms", 5000); // by default 5s
// %s
// 0, means that the producer never waits for an acknowledgement from the broker
// 1, means that the producer gets an acknowledgement after the leader replica has received the data
//-1, means that the producer gets an acknowledgement after all in-sync replicas have received the data
props.put("%s", "1"); // 1 is enough in most cases
producer = new Producer<String, String>(new ProducerConfig(props));
}
void produce() {
int messageNo = 1000;
final int COUNT = 10000;
while (messageNo < COUNT) {
String key = String.valueOf(messageNo);
String data = "hello kafka message " + key;
producer.send(new KeyedMessage<String, String>(TOPIC, key, data));
messageNo ++;
}
}
public static void main(String[] args) {
new KafkaProducer().produce();
}
}
`,
color.Cyan("metadata.broker.list"),
color.Cyan("request.required.acks"),
color.Cyan("request.required.acks"))
}
func (*Sample) offsetCommitFetchSample() string {
return fmt.Sprintf(`
KafkaConsumer<K, V> consumer = new KafkaConsumer<K, V>(properties);
...
TopicPartition p0 = new TopicPartition("topic1", 0);
TopicPartition p1 = new TopicPartition("topic1", 1);
%s
Map<TopicPartition, Long> offsets = new LinkedHashMap<TopicPartition, Long>();
offsets.Put(p0, 123L);
offsets.Put(p1, 4564L);
%s
consumer.commit(offsets, CommitType.SYNC);
%s
long committedOffset = consumer.committed(p1);
`,
color.Cyan("consumer.subscribe(p0, p1);"),
color.Cyan("// commit offsets"),
color.Cyan("//fetch offsets"))
}
func (this *Kateway) Run(args []string) (exitCode int) {
cmdFlags := flag.NewFlagSet("kateway", flag.ContinueOnError)
cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
cmdFlags.StringVar(&this.zone, "z", "", "")
cmdFlags.BoolVar(&this.configMode, "cf", false, "")
cmdFlags.StringVar(&this.id, "id", "", "")
cmdFlags.BoolVar(&this.install, "i", false, "")
cmdFlags.BoolVar(&this.longFmt, "l", false, "")
cmdFlags.StringVar(&this.configOption, "option", "", "")
cmdFlags.BoolVar(&this.versionOnly, "ver", false, "")
cmdFlags.BoolVar(&this.flameGraph, "flame", false, "")
cmdFlags.StringVar(&this.logLevel, "loglevel", "", "")
cmdFlags.StringVar(&this.visualLog, "visualog", "", "")
cmdFlags.BoolVar(&this.showZkNodes, "zk", false, "")
cmdFlags.BoolVar(&this.checkup, "checkup", false, "")
cmdFlags.BoolVar(&this.benchmark, "bench", false, "")
cmdFlags.StringVar(&this.benchmarkMaster, "master", "", "")
cmdFlags.BoolVar(&this.pub, "pub", false, "")
cmdFlags.BoolVar(&this.sub, "sub", false, "")
cmdFlags.BoolVar(&this.benchmarkAsync, "async", false, "")
cmdFlags.BoolVar(&this.curl, "curl", false, "")
if err := cmdFlags.Parse(args); err != nil {
return 2
}
if this.benchmark {
if validateArgs(this, this.Ui).
require("-z").
requireAdminRights("-z").
invalid(args) {
return 2
}
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
zone := ctx.Zone(zkzone.Name())
this.benchApp = zone.SmokeApp
this.benchSecret = zone.SmokeSecret
this.benchTopic = zone.SmokeTopic
this.benchVer = zone.SmokeTopicVersion
this.benchPubEndpoint = zone.PubEndpoint
if this.id != "" {
kws, err := zkzone.KatewayInfos()
swallow(err)
for _, kw := range kws {
if kw.Id == this.id {
this.benchPubEndpoint = kw.PubAddr
break
}
}
}
this.benchId = fmt.Sprintf("%s-%s", ctx.Hostname(), strings.Replace(uuid.New(), "-", "", -1))
this.runBenchmark(zkzone)
return
}
if this.flameGraph {
if validateArgs(this, this.Ui).
require("-z", "-id").
requireAdminRights("-z").
invalid(args) {
return 2
}
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
this.generateFlameGraph(zkzone)
return
}
if this.visualLog != "" {
this.doVisualize()
return
}
if this.pub {
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
this.runPub(zkzone)
return
}
if this.sub {
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
this.runSub(zkzone)
return
}
if this.install {
if validateArgs(this, this.Ui).
require("-z").
invalid(args) {
return 2
}
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
this.installGuide(zkzone)
return
}
if this.configOption != "" {
this.configMode = true
}
if this.configMode {
if validateArgs(this, this.Ui).
require("-z").
requireAdminRights("-z").
invalid(args) {
return 2
}
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
if this.logLevel != "" {
if this.id != "" {
kw := zkzone.KatewayInfoById(this.id)
if kw == nil {
panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id))
}
this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel))
} else {
// apply on all kateways
kws, _ := zkzone.KatewayInfos()
for _, kw := range kws {
this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel))
}
}
}
if this.configOption != "" {
parts := strings.SplitN(this.configOption, "=", 2)
if len(parts) != 2 {
this.Ui.Error("usage: key=value")
return
}
k, v := parts[0], parts[1]
if this.id != "" {
kw := zkzone.KatewayInfoById(this.id)
if kw == nil {
panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id))
}
this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v))
} else {
// apply on all kateways
kws, _ := zkzone.KatewayInfos()
for _, kw := range kws {
this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v))
}
}
}
return
}
if this.checkup {
if this.zone == "" {
forAllSortedZones(func(zkzone *zk.ZkZone) {
this.runCheckup(zkzone)
})
return
}
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
this.runCheckup(zkzone)
return
}
if this.showZkNodes {
this.Ui.Output(fmt.Sprintf(`%s pubsub manager db dsn
%s job db cluster config
%s turn off webhook dir`,
color.Green("%-50s", zk.KatewayMysqlPath),
color.Green("%-50s", zk.PubsubJobConfig),
color.Green("%-50s", zk.PubsubWebhooksOff)))
return
}
// display mode
lines := make([]string, 0)
header := "Zone|Id|Ip|Pprof|Build|Cpu|Heap|Obj|Go|P/S|hhIn/hhOut|Uptime"
lines = append(lines, header)
forSortedZones(func(zkzone *zk.ZkZone) {
if this.zone != "" && zkzone.Name() != this.zone {
return
}
if !this.versionOnly {
mysqlDsn, err := zkzone.KatewayMysqlDsn()
if err != nil {
this.Ui.Warn(fmt.Sprintf("kateway[%s] mysql DSN not set on zk yet", zkzone.Name()))
this.Ui.Output(fmt.Sprintf("e,g. %s -> pubsub:pubsub@tcp(10.77.135.217:10010)/pubsub?charset=utf8&timeout=10s",
zk.KatewayMysqlPath))
} else {
this.Ui.Output(fmt.Sprintf("zone[%s] manager db: %s",
color.Cyan(zkzone.Name()), mysqlDsn))
}
}
kateways, err := zkzone.KatewayInfos()
if err != nil {
if err == zklib.ErrNoNode {
this.Ui.Output("no kateway running")
return
} else {
swallow(err)
}
}
for _, kw := range kateways {
if this.id != "" && this.id != kw.Id {
continue
}
statusMap, _ := this.getKatewayStatusMap(kw.ManAddr)
logLevel, _ := statusMap["loglevel"].(string)
heapSize, _ := statusMap["heap"].(string)
heapObjs, _ := statusMap["objects"].(string)
pubConn, _ := statusMap["pubconn"].(string)
hhAppendN, _ := statusMap["hh_appends"].(string)
hhDeliverN, _ := statusMap["hh_delivers"].(string)
subConn, _ := statusMap["subconn"].(string)
goN, _ := statusMap["goroutines"].(string)
if this.versionOnly {
pprofAddr := kw.DebugAddr
if len(pprofAddr) > 0 && pprofAddr[0] == ':' {
pprofAddr = kw.Ip + pprofAddr
}
pprofAddr = fmt.Sprintf("%s/debug/pprof/", pprofAddr)
lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s/%s|%s|%s|%s|%s|%s/%s|%s/%s|%s",
zkzone.Name(),
kw.Id, kw.Ip,
pprofAddr, kw.Build, kw.BuiltAt,
kw.Cpu,
heapSize, heapObjs,
goN,
pubConn, subConn, hhAppendN, hhDeliverN,
gofmt.PrettySince(kw.Ctime)))
continue
}
this.Ui.Info(fmt.Sprintf("id:%-2s host:%s cpu:%-2s up:%s",
kw.Id, kw.Host, kw.Cpu,
gofmt.PrettySince(kw.Ctime)))
this.Ui.Output(fmt.Sprintf(" ver: %s\n arch: %s\n build: %s\n built: %s\n log: %s\n pub: %s\n sub: %s\n man: %s\n dbg: %s",
kw.Ver,
kw.Arch,
color.Red(kw.Build),
kw.BuiltAt,
logLevel,
kw.PubAddr,
kw.SubAddr,
kw.ManAddr,
kw.DebugAddr,
))
if this.longFmt {
this.Ui.Output(" full status:")
this.Ui.Output(this.getKatewayStatus(kw.ManAddr))
}
}
})
if this.versionOnly && len(lines) > 1 {
fmt.Println(columnize.SimpleFormat(lines))
}
return
}
func (this *Host) diagnose() {
clusters := make([]string, 0)
leadingTopics := make(map[string][]int32) // topic:partitionID
msgs := make(map[string]int64)
// check if it is registered
zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
zkcluster := zkzone.NewCluster(cluster)
for _, broker := range liveBrokers {
if broker.Host != this.host {
continue
}
this.Ui.Output(broker.String())
clusters = append(clusters, cluster)
kfk, err := sarama.NewClient([]string{broker.Addr()}, saramaConfig())
if err != nil {
this.Ui.Error(err.Error())
continue
}
topics, err := kfk.Topics()
swallow(err)
for _, topic := range topics {
partions, err := kfk.WritablePartitions(topic)
swallow(err)
for _, partitionID := range partions {
leader, err := kfk.Leader(topic, partitionID)
swallow(err)
host, _, err := net.SplitHostPort(leader.Addr())
swallow(err)
if host != this.host {
continue
}
latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
swallow(err)
oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
swallow(err)
msgs[topic] += (latestOffset - oldestOffset)
if _, present := leadingTopics[topic]; !present {
leadingTopics[topic] = make([]int32, 0)
}
leadingTopics[topic] = append(leadingTopics[topic], partitionID)
}
}
kfk.Close()
}
registeredBrokers := zkcluster.RegisteredInfo().Roster
for _, b := range registeredBrokers {
if b.Host == this.host {
this.Ui.Output(fmt.Sprintf("registered in cluster: %s", cluster))
}
}
})
this.Ui.Output(color.Cyan("clusters\n%s", strings.Repeat("-", 80)))
for _, c := range clusters {
this.Ui.Output(c)
}
this.Ui.Output(color.Cyan("clusters\n%s", strings.Repeat("-", 80)))
for t, ps := range leadingTopics {
this.Ui.Output(fmt.Sprintf("%35s M:%-10d P:%+v", t, msgs[t], ps))
}
}
func (this *Checkup) Run(args []string) (exitCode int) {
var cmd cli.Command
if false {
this.Ui.Output(color.Cyan("checking zookeepeer\n%s", strings.Repeat("-", 80)))
cmd = &Zookeeper{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(append(args, "-c", "srvr"))
this.Ui.Output("")
}
this.Ui.Output(color.Cyan("ping all brokers\n%s", strings.Repeat("-", 80)))
cmd = &Ping{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(append(args, "-p"))
this.Ui.Output("")
this.Ui.Output(color.Cyan("checking registered brokers are alive\n%s", strings.Repeat("-", 80)))
cmd = &Clusters{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(append(args, "-verify"))
this.Ui.Output("")
this.Ui.Output(color.Cyan("checking offline brokers\n%s", strings.Repeat("-", 80)))
cmd = &Brokers{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(append(args, "-stale"))
this.Ui.Output("")
this.Ui.Output(color.Cyan("checking under replicated brokers\n%s", strings.Repeat("-", 80)))
cmd = &UnderReplicated{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(args)
this.Ui.Output("")
this.Ui.Output(color.Cyan("checking kguard\n%s", strings.Repeat("-", 80)))
cmd = &Kguard{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(args)
this.Ui.Output("")
this.Ui.Output(color.Cyan("checking problematic lag consumers\n%s", strings.Repeat("-", 80)))
cmd = &Lags{
Ui: this.Ui,
Cmd: this.Cmd,
}
cmd.Run(append(args, "-p"))
this.Ui.Output("")
this.Ui.Output("Did you find something wrong?")
return
}
func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) {
echoBuffer := func(lines []string) {
for _, l := range lines {
this.Ui.Output(l)
}
}
linesInTopicMode := make([]string, 0)
if this.verbose {
linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode)
}
// get all alive brokers within this cluster
brokers := zkcluster.Brokers()
if len(brokers) == 0 {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ",
color.Red("%s empty brokers", zkcluster.Name())), linesInTopicMode)
echoBuffer(linesInTopicMode)
return
}
if this.verbose {
sortedBrokerIds := make([]string, 0, len(brokers))
for brokerId, _ := range brokers {
sortedBrokerIds = append(sortedBrokerIds, brokerId)
}
sort.Strings(sortedBrokerIds)
for _, brokerId := range sortedBrokerIds {
if this.ipInNumber {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
color.Green(brokerId), brokers[brokerId]), linesInTopicMode)
} else {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
color.Green(brokerId), brokers[brokerId].NamedString()), linesInTopicMode)
}
}
}
kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig())
if err != nil {
if this.verbose {
linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%+v %s", " ",
zkcluster.BrokerList(), err.Error()), linesInTopicMode)
}
return
}
defer kfk.Close()
topics, err := kfk.Topics()
swallow(err)
if len(topics) == 0 {
if this.topicPattern == "" && this.verbose {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ",
color.Magenta("no topics")), linesInTopicMode)
echoBuffer(linesInTopicMode)
}
return
}
sortedTopics := make([]string, 0, len(topics))
for _, t := range topics {
sortedTopics = append(sortedTopics, t)
}
sort.Strings(sortedTopics)
topicsCtime := zkcluster.TopicsCtime()
hasTopicMatched := false
for _, topic := range sortedTopics {
if !patternMatched(topic, this.topicPattern) {
continue
}
if this.since > 0 && time.Since(topicsCtime[topic]) > this.since {
continue
}
this.topicN++
hasTopicMatched = true
if this.verbose {
linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Cyan(topic), linesInTopicMode)
}
// get partitions and check if some dead
alivePartitions, err := kfk.WritablePartitions(topic)
swallow(err)
partions, err := kfk.Partitions(topic)
swallow(err)
if len(alivePartitions) != len(partions) {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %s P: %s/%+v",
zkcluster.Name(), color.Cyan("%-50s", topic), color.Red("partial dead"), color.Green("%+v", alivePartitions), partions), linesInTopicMode)
}
replicas, err := kfk.Replicas(topic, partions[0])
if err != nil {
this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partions[0], err))
}
this.partitionN += len(partions)
if !this.verbose {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR %s",
zkcluster.Name(),
color.Cyan("%-50s", topic),
len(partions), len(replicas),
gofmt.PrettySince(topicsCtime[topic])), linesInTopicMode)
continue
}
for _, partitionID := range alivePartitions {
leader, err := kfk.Leader(topic, partitionID)
swallow(err)
replicas, err := kfk.Replicas(topic, partitionID)
if err != nil {
this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partitionID, err))
}
isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)
isrMtimeSince := gofmt.PrettySince(isrMtime)
if time.Since(isrMtime).Hours() < 24 {
// ever out of sync last 24h
isrMtimeSince = color.Magenta(isrMtimeSince)
}
underReplicated := false
if len(isr) != len(replicas) {
underReplicated = true
}
latestOffset, err := kfk.GetOffset(topic, partitionID,
sarama.OffsetNewest)
swallow(err)
oldestOffset, err := kfk.GetOffset(topic, partitionID,
sarama.OffsetOldest)
swallow(err)
if this.count > 0 && (latestOffset-oldestOffset) < this.count {
continue
}
this.totalMsgs += latestOffset - oldestOffset
this.totalOffsets += latestOffset
if !underReplicated {
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%+v Offset:%16s - %-16s Num:%-15s %s-%s",
partitionID,
color.Green("%d", leader.ID()), replicas, isr,
gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset),
gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode)
} else {
// use red for alert
linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%s Offset:%16s - %-16s Num:%-15s %s-%s",
partitionID,
color.Green("%d", leader.ID()), replicas, color.Red("%+v", isr),
gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset),
gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode)
}
}
}
if this.topicPattern != "" {
if hasTopicMatched {
echoBuffer(linesInTopicMode)
}
} else {
echoBuffer(linesInTopicMode)
}
}
func (*Sample) consumeSample() string {
return fmt.Sprintf(`
public class KafkaConsumer {
private final ConsumerConnector consumer;
private KafkaConsumer() {
Properties props = new Properties();
props.put("%s", "zk2181a.wdds.zk.com:2181,zk2181b.wdds.zk.com:2181,zk2181c.wdds.zk.com:2181/kafka");
props.put("%s", "group1");
props.put("zookeeper.session.timeout.ms", "4000");
props.put("zookeeper.sync.time.ms", "200");
props.put("auto.commit.interval.ms", "60000"); // 1m
//props.put("auto.offset.reset", "smallest"); // largest | smallest
props.put("serializer.class", "kafka.serializer.StringEncoder");
ConsumerConfig config = new ConsumerConfig(props);
consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config);
}
public void shutdown() {
if (consumer != null) {
consumer.shutdown();
}
}
void consume(String topic, int %s) {
// %s
// %s
// %s
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
consumer.shutdown();
}
});
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(topic, %s);
StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
Map<String, List<KafkaStream<String, String>>> consumerMap =
consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
KafkaStream<String, String> stream = consumerMap.get(topic).get(0);
ConsumerIterator<String, String> it = stream.iterator();
while (it.hasNext()) {
// consumer.commitOffsets(); // manually commit offsets
System.out.println(it.next().message());
}
}
public static void main(String[] args) {
new KafkaConsumer().consume();
}
}
`,
color.Cyan("zookeeper.connect"),
color.Cyan("group.id"),
color.Green("threads"),
color.Red("VERY important!"),
color.Red("graceful shutdown the consumer group to commit consumed offset"),
color.Red("avoid consuming duplicated message when restarting the same consumer group"),
color.Green("threads"))
}