func (this *pubStore) Start() (err error) {
if ctx.KafkaHome() == "" {
return fmt.Errorf("empty kafka_home in ~/.gafka.cf")
}
if !gio.DirExists(ctx.KafkaHome()) {
return fmt.Errorf("kafka not installed in %s, run 'gk deploy -kfkonly'", ctx.KafkaHome())
}
// warmup: create pools according the current kafka topology
for _, cluster := range meta.Default.ClusterNames() {
this.pubPools[cluster] = newPubPool(this, cluster,
meta.Default.BrokerList(cluster), this.pubPoolsCapcity)
}
this.wg.Add(1)
go func() {
defer this.wg.Done()
for {
select {
case <-meta.Default.RefreshEvent():
this.doRefresh()
case <-this.shutdownCh:
log.Trace("pub store[%s] stopped", this.Name())
return
}
}
}()
return
}
func (this *Migrate) executeReassignment() {
/*
1. kafka-reassign-partitions.sh write /admin/reassign_partitions
2. controller listens to the path above
3. For each topic partition, the controller does the following:
3.1. Start new replicas in RAR – AR (RAR = Reassigned Replicas, AR = original list of Assigned Replicas)
3.2. Wait until new replicas are in sync with the leader
3.3. If the leader is not in RAR, elect a new leader from RAR
3.4 4. Stop old replicas AR – RAR
3.5. Write new AR
3.6. Remove partition from the /admin/reassign_partitions path
*/
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-reassign-partitions.sh", ctx.KafkaHome()),
fmt.Sprintf("--zookeeper %s", this.zkcluster.ZkConnectAddr()),
fmt.Sprintf("--reassignment-json-file %s", reassignNodeFilename),
fmt.Sprintf("--execute"),
)
err := cmd.Open()
if err != nil {
return
}
defer cmd.Close()
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
this.Ui.Output(color.Yellow(scanner.Text()))
}
}
func (this *Topics) resetTopicConfig(zkcluster *zk.ZkCluster, topic string) {
zkAddrs := zkcluster.ZkConnectAddr()
key := "retention.ms"
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-topics.sh", ctx.KafkaHome()),
fmt.Sprintf("--zookeeper %s", zkAddrs),
fmt.Sprintf("--alter"),
fmt.Sprintf("--topic %s", topic),
fmt.Sprintf("--deleteConfig %s", key),
)
err := cmd.Open()
swallow(err)
defer cmd.Close()
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
output := make([]string, 0)
for scanner.Scan() {
output = append(output, scanner.Text())
}
swallow(scanner.Err())
path := zkcluster.GetTopicConfigPath(topic)
this.Ui.Info(path)
for _, line := range output {
this.Ui.Output(line)
}
}
func (this *Partition) addPartition(zkAddrs string, topic string, partitions int) error {
log.Info("adding partitions to topic: %s", topic)
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-topics.sh", ctx.KafkaHome()),
fmt.Sprintf("--zookeeper %s", zkAddrs),
fmt.Sprintf("--alter"),
fmt.Sprintf("--topic %s", topic),
fmt.Sprintf("--partitions %d", partitions),
)
err := cmd.Open()
if err != nil {
return err
}
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
this.Ui.Output(color.Yellow(scanner.Text()))
}
err = scanner.Err()
if err != nil {
return err
}
cmd.Close()
log.Info("added partitions to topic: %s", topic)
return nil
}
func (this *ZkCluster) AlterTopic(topic string, ts *sla.TopicSla) (output []string, err error) {
zkAddrs := this.ZkConnectAddr()
args := []string{
fmt.Sprintf("--zookeeper %s", zkAddrs),
fmt.Sprintf("--alter"),
fmt.Sprintf("--topic %s", topic),
}
configs := ts.DumpForAlterTopic()
if len(configs) == 0 {
err = errors.New("no alter topic configs")
return
}
args = append(args, configs...)
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-topics.sh", ctx.KafkaHome()),
args...)
if err = cmd.Open(); err != nil {
return
}
defer cmd.Close()
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
output = make([]string, 0)
for scanner.Scan() {
output = append(output, scanner.Text())
}
if err = scanner.Err(); err != nil {
return
}
return
}
func (this *ZkCluster) DeleteTopic(topic string) (output []string, err error) {
zkAddrs := this.ZkConnectAddr()
args := []string{
fmt.Sprintf("--zookeeper %s", zkAddrs),
fmt.Sprintf("--delete"),
fmt.Sprintf("--topic %s", topic),
}
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-topics.sh", ctx.KafkaHome()), args...)
if err = cmd.Open(); err != nil {
return
}
defer cmd.Close()
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
output = make([]string, 0)
for scanner.Scan() {
output = append(output, scanner.Text())
}
if err = scanner.Err(); err != nil {
return
}
return
}
func (this *Deploy) Help() string {
help := fmt.Sprintf(`
Usage: %s deploy -z zone -c cluster [options]
%s
Options:
-kfkonly
Only install kafka runtime on localhost.
-influx host:port
InfluxDB server address used for kafka metrics reporter.
If empty, InfluxDB metrics reporter will be disabled.
-uninstall base dir
Uninstall a kafka broker on localhost.
-demo
Demonstrate how to use this command.
-dryrun
Default is true.
-root dir
Root directory of the kafka broker.
Defaults to /var/wd
-ip addr
Advertised host name of this new broker.
-port port
Tcp port the broker will listen on.
-broker.id id
-user runAsUser
The deployed kafka broker will run as this user.
Defaults to sre
-ver [2.10-0.8.1.1|2.10-0.8.2.2]
Defaults to 2.10-0.8.2.2
-kafka.base dir
Kafka installation prefix dir.
Defaults to %s
-log.dirs dirs
A comma seperated list of directories under which to store log files.
`, this.Cmd, this.Synopsis(), ctx.KafkaHome())
return strings.TrimSpace(help)
}
func (this *Rebalance) executeReassignment() {
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-preferred-replica-election.sh", ctx.KafkaHome()),
fmt.Sprintf("--zookeeper %s", this.zkcluster.ZkConnectAddr()),
fmt.Sprintf("--path-to-json-file %s", preferredReplicaJsonFile),
)
err := cmd.Open()
if err != nil {
return
}
defer cmd.Close()
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
this.Ui.Output(color.Yellow(scanner.Text()))
}
}
func (this *Migrate) verify() {
cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-reassign-partitions.sh", ctx.KafkaHome()),
fmt.Sprintf("--zookeeper %s", this.zkcluster.ZkConnectAddr()),
fmt.Sprintf("--reassignment-json-file %s", reassignNodeFilename),
fmt.Sprintf("--verify"),
)
err := cmd.Open()
if err != nil {
return
}
defer cmd.Close()
scanner := bufio.NewScanner(cmd.Reader())
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
if strings.Contains(scanner.Text(), "successfully") {
this.Ui.Info(scanner.Text())
} else {
this.Ui.Warn(scanner.Text())
}
}
}
// TODO
// 1. broker id assignment
// 2. port assignment
func (this *Deploy) Run(args []string) (exitCode int) {
cmdFlags := flag.NewFlagSet("deploy", flag.ContinueOnError)
cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
cmdFlags.StringVar(&this.zone, "z", "", "")
cmdFlags.StringVar(&this.cluster, "c", "", "")
cmdFlags.StringVar(&this.kafkaBaseDir, "kafka.base", ctx.KafkaHome(), "")
cmdFlags.StringVar(&this.brokerId, "broker.id", "", "")
cmdFlags.StringVar(&this.tcpPort, "port", "", "")
cmdFlags.StringVar(&this.rootPah, "root", "/var/wd", "")
cmdFlags.StringVar(&this.ip, "ip", "", "")
cmdFlags.StringVar(&this.logDirs, "log.dirs", "", "")
cmdFlags.StringVar(&this.runAs, "user", "sre", "")
cmdFlags.StringVar(&this.uninstall, "uninstall", "", "")
cmdFlags.BoolVar(&this.demoMode, "demo", false, "")
cmdFlags.BoolVar(&this.installKafkaOnly, "kfkonly", false, "")
cmdFlags.BoolVar(&this.dryRun, "dryrun", true, "")
cmdFlags.StringVar(&this.influxDbAddr, "influx", "", "")
cmdFlags.StringVar(&this.kafkaVer, "ver", "2.10-0.8.2.2", "")
if err := cmdFlags.Parse(args); err != nil {
return 1
}
if this.uninstall != "" {
serverProperties := fmt.Sprintf("%s/config/server.properties", this.uninstall)
lines, err := gio.ReadLines(serverProperties)
if err != nil {
this.Ui.Error(err.Error())
return 2
}
var logDirs []string
for _, line := range lines {
if strings.HasPrefix(line, "log.dirs") {
parts := strings.SplitN(line, "=", 2)
logDirs = strings.Split(parts[1], ",")
break
}
}
if len(logDirs) == 0 {
this.Ui.Error("empty log.dirs")
return 2
}
for _, logDir := range logDirs {
this.Ui.Output(fmt.Sprintf("rm -rf %s", logDir))
}
name := filepath.Base(this.uninstall)
this.Ui.Output(fmt.Sprintf("chkconfig --del %s", name))
this.Ui.Output(fmt.Sprintf("rm -f /etc/init.d/%s", name))
this.Ui.Output(fmt.Sprintf("rm -rf %s", this.uninstall))
return 0
}
if !ctx.CurrentUserIsRoot() {
this.Ui.Error("requires root priviledges!")
return 1
}
if !strings.HasSuffix(this.kafkaBaseDir, this.kafkaVer) {
this.Ui.Error(fmt.Sprintf("kafka.base[%s] does not match ver[%s]",
this.kafkaBaseDir, this.kafkaVer))
return 1
}
if this.installKafkaOnly {
this.installKafka()
return
}
if validateArgs(this, this.Ui).
require("-z", "-c").
invalid(args) {
return 2
}
this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
clusers := this.zkzone.Clusters()
zkchroot, present := clusers[this.cluster]
if !present {
this.Ui.Error(fmt.Sprintf("run 'gk clusters -z %s -add %s -p $zkchroot' first!",
this.zone, this.cluster))
return 1
}
var err error
this.userInfo, err = user.Lookup(this.runAs)
swallow(err)
if this.demoMode {
this.demo()
return
}
if validateArgs(this, this.Ui).
require("-broker.id", "-port", "-ip", "-log.dirs").
invalid(args) {
return 2
}
if this.dryRun {
this.Ui.Output(fmt.Sprintf("mkdir %s/logs and chown to %s",
this.instanceDir(), this.runAs))
}
err = os.MkdirAll(fmt.Sprintf("%s/logs", this.instanceDir()), 0755)
swallow(err)
chown(fmt.Sprintf("%s/logs", this.instanceDir()), this.userInfo)
invalidDir := this.validateLogDirs(this.logDirs)
if invalidDir != "" {
this.Ui.Error(fmt.Sprintf("%s in log.dirs not exists!", invalidDir))
return 2
}
// prepare the root directory
this.rootPah = strings.TrimSuffix(this.rootPah, "/")
if this.dryRun {
this.Ui.Output(fmt.Sprintf("mkdir %s/bin and chown to %s",
this.instanceDir(), this.runAs))
}
err = os.MkdirAll(fmt.Sprintf("%s/bin", this.instanceDir()), 0755)
swallow(err)
chown(fmt.Sprintf("%s/bin", this.instanceDir()), this.userInfo)
if this.dryRun {
this.Ui.Output(fmt.Sprintf("mkdir %s/config and chown to %s",
this.instanceDir(), this.runAs))
}
err = os.MkdirAll(fmt.Sprintf("%s/config", this.instanceDir()), 0755)
swallow(err)
chown(fmt.Sprintf("%s/config", this.instanceDir()), this.userInfo)
type templateVar struct {
KafkaBase string
BrokerId string
TcpPort string
Ip string
User string
ZkChroot string
ZkAddrs string
InstanceDir string
LogDirs string
IoThreads string
NetworkThreads string
InfluxReporterEnabled string
InfluxDbHost string
InfluxDbPort string
}
if this.influxDbAddr != "" {
this.influxDbHost, this.influxdbPort, err = net.SplitHostPort(this.influxDbAddr)
if err != nil {
this.Ui.Error(err.Error())
return 2
}
if this.influxDbHost == "" || this.influxdbPort == "" {
this.Ui.Error("empty influxdb host or port")
return 2
}
}
influxReporterEnabled := "false"
if this.influxDbHost != "" {
influxReporterEnabled = "true"
}
data := templateVar{
ZkAddrs: this.zkzone.ZkAddrs(),
ZkChroot: zkchroot,
KafkaBase: this.kafkaBaseDir,
BrokerId: this.brokerId,
Ip: this.ip,
InstanceDir: this.instanceDir(),
User: this.runAs,
TcpPort: this.tcpPort,
LogDirs: this.logDirs,
InfluxReporterEnabled: influxReporterEnabled,
InfluxDbHost: this.influxDbHost,
InfluxDbPort: this.influxdbPort,
}
data.IoThreads = strconv.Itoa(3 * len(strings.Split(data.LogDirs, ",")))
networkThreads := ctx.NumCPU() / 2
if networkThreads < 2 {
networkThreads = 2
}
data.NetworkThreads = strconv.Itoa(networkThreads) // TODO not used yet
// create the log.dirs directory and chown to sre
logDirs := strings.Split(this.logDirs, ",")
for _, logDir := range logDirs {
if this.dryRun {
this.Ui.Output(fmt.Sprintf("mkdir %s and chown to %s",
logDir, this.runAs))
}
swallow(os.MkdirAll(logDir, 0755))
chown(logDir, this.userInfo)
}
// package the kafka runtime together
if !gio.DirExists(this.kafkaLibDir()) {
this.installKafka()
}
// bin
writeFileFromTemplate("template/bin/kafka-topics.sh",
fmt.Sprintf("%s/bin/kafka-topics.sh", this.instanceDir()), 0755, nil, this.userInfo)
writeFileFromTemplate("template/bin/kafka-reassign-partitions.sh",
fmt.Sprintf("%s/bin/kafka-reassign-partitions.sh", this.instanceDir()), 0755, nil, this.userInfo)
writeFileFromTemplate("template/bin/kafka-preferred-replica-election.sh",
fmt.Sprintf("%s/bin/kafka-preferred-replica-election.sh", this.instanceDir()), 0755, nil, this.userInfo)
writeFileFromTemplate("template/bin/kafka-run-class.sh",
fmt.Sprintf("%s/bin/kafka-run-class.sh", this.instanceDir()), 0755, data, this.userInfo)
writeFileFromTemplate("template/bin/kafka-server-start.sh",
fmt.Sprintf("%s/bin/kafka-server-start.sh", this.instanceDir()), 0755, data, this.userInfo)
writeFileFromTemplate("template/bin/setenv.sh",
fmt.Sprintf("%s/bin/setenv.sh", this.instanceDir()), 0755, data, this.userInfo)
// /etc/init.d/
writeFileFromTemplate("template/init.d/kafka",
fmt.Sprintf("/etc/init.d/%s", this.clusterName()), 0755, data, nil)
// config
writeFileFromTemplate("template/config/server.properties",
fmt.Sprintf("%s/config/server.properties", this.instanceDir()), 0644, data, this.userInfo)
writeFileFromTemplate("template/config/log4j.properties",
fmt.Sprintf("%s/config/log4j.properties", this.instanceDir()), 0644, data, this.userInfo)
this.Ui.Warn(fmt.Sprintf("NOW, please run the following command:"))
this.Ui.Output(color.Red("confirm log.retention.hours"))
this.Ui.Output(color.Red("chkconfig --add %s", this.clusterName()))
this.Ui.Output(color.Red("/etc/init.d/%s start", this.clusterName()))
return
}