func (this *Redis) queryInfluxDB(cmd string) (res []client.Result, err error) { addr := ctx.Zone(this.zone).InfluxAddr if !strings.HasPrefix(addr, "http") { addr = fmt.Sprintf("http://%s", addr) } cli, err := client.NewHTTPClient(client.HTTPConfig{ Addr: addr, Username: "", Password: "", }) if err != nil { return } q := client.Query{ Command: cmd, Database: "kfk_prod", } if response, err := cli.Query(q); err == nil { if response.Error() != nil { return res, response.Error() } res = response.Results } else { return res, err } return res, nil }
func (this *Kateway) runSub(zkzone *zk.ZkZone) { zone := ctx.Zone(zkzone.Name()) cf := api.DefaultConfig(zone.SmokeApp, zone.SmokeSecret) cf.Sub.Endpoint = zone.SubEndpoint cli := api.NewClient(cf) t1 := time.Now() err := cli.Sub(api.SubOption{ AppId: zone.SmokeHisApp, Topic: zone.SmokeTopic, Ver: zone.SmokeTopicVersion, Group: zone.SmokeGroup, AutoClose: false, }, func(statusCode int, subMsg []byte) error { now := time.Now() var e error if statusCode != http.StatusOK { e = fmt.Errorf("unexpected http status: %s, body:%s", http.StatusText(statusCode), string(subMsg)) } this.Ui.Output(fmt.Sprintf("-> %s: %s %v", now.Sub(t1), string(subMsg), e)) time.Sleep(time.Millisecond * 100) t1 = time.Now() return e }) if err != nil { this.Ui.Error(err.Error()) } }
func (this *Kateway) installGuide(zkzone *zk.ZkZone) { this.Ui.Output(color.Red("manager db GRANT access rights to this ip")) this.Ui.Output(color.Red("gk deploy -kfkonly")) this.Ui.Output("") this.Ui.Output("mkdir -p /var/wd/kateway/sbin") this.Ui.Output("cd /var/wd/kateway") kateways, err := zkzone.KatewayInfos() swallow(err) nextId := 1 for _, kw := range kateways { id, _ := strconv.Atoi(kw.Id) if nextId < id { nextId = id } } nextId++ zone := ctx.Zone(this.zone) influxAddr := zone.InfluxAddr if influxAddr != "" && !strings.HasPrefix(influxAddr, "http://") { influxAddr = "http://" + influxAddr } var influxInfo string if influxAddr != "" { influxInfo = "-influxdbaddr " + influxAddr } this.Ui.Output(fmt.Sprintf(`nohup ./sbin/kateway -zone prod -id %d -debughttp ":10194" -level trace -log kateway.log -crashlog panic %s &`, nextId, influxInfo)) this.Ui.Output("") this.Ui.Output("yum install -y logstash") this.Ui.Output("/etc/logstash/conf.d/kateway.conf") this.Ui.Output(strings.TrimSpace(fmt.Sprintf(` input { file { path => "/var/wd/kateway/kateway.log" type => "kateway" } file { path => "/var/wd/kateway/panic" type => "kateway_panic" } } output { kafka { bootstrap_servers => "%s:11003,%s:11003" topic_id => "pubsub_log" } } `, color.Red("k11003a.mycorp.kfk.com"), color.Red("k11003b.mycorp.kfk.com")))) this.Ui.Output("") this.Ui.Output("chkconfig --add logstash") this.Ui.Output("/etc/init.d/logstash start") }
func (this *Kateway) runPub(zkzone *zk.ZkZone) { zone := ctx.Zone(zkzone.Name()) cf := api.DefaultConfig(zone.SmokeApp, zone.SmokeSecret) cf.Pub.Endpoint = zone.PubEndpoint cli := api.NewClient(cf) for { now := time.Now() pubMsg := fmt.Sprintf("gk kateway -pub smoke test msg: [%s]", now) err := cli.Pub("", []byte(pubMsg), api.PubOption{ Topic: zone.SmokeTopic, Ver: zone.SmokeTopicVersion, }) this.Ui.Output(fmt.Sprintf("<- %s: %s %v", time.Since(now), pubMsg, err)) time.Sleep(time.Millisecond * 100) } }
func (this *Haproxy) Run(args []string) (exitCode int) { var topMode bool cmdFlags := flag.NewFlagSet("haproxy", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone, "z", ctx.DefaultZone(), "") cmdFlags.BoolVar(&topMode, "top", true, "") if err := cmdFlags.Parse(args); err != nil { return 1 } zone := ctx.Zone(this.zone) if topMode { header, _ := this.getStats(zone.HaProxyStatsUri[0]) t := top.New(header, "%8s %4s %15s %15s %8s %6s %8s %10s %8s %8s %5s %7s %9s %6s") go func() { for { rows := make([]string, 0) for _, uri := range zone.HaProxyStatsUri { _, r := this.getStats(uri) rows = append(rows, r...) } t.Refresh(rows) time.Sleep(time.Second * 5) } }() if err := t.Start(); err != nil { panic(err) } } else { for _, uri := range zone.HaProxyStatsUri { this.fetchStats(uri) } } return }
func (this *Kateway) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("kateway", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone, "z", "", "") cmdFlags.BoolVar(&this.configMode, "cf", false, "") cmdFlags.StringVar(&this.id, "id", "", "") cmdFlags.BoolVar(&this.install, "i", false, "") cmdFlags.BoolVar(&this.longFmt, "l", false, "") cmdFlags.StringVar(&this.configOption, "option", "", "") cmdFlags.BoolVar(&this.versionOnly, "ver", false, "") cmdFlags.BoolVar(&this.flameGraph, "flame", false, "") cmdFlags.StringVar(&this.logLevel, "loglevel", "", "") cmdFlags.StringVar(&this.visualLog, "visualog", "", "") cmdFlags.BoolVar(&this.showZkNodes, "zk", false, "") cmdFlags.BoolVar(&this.checkup, "checkup", false, "") cmdFlags.BoolVar(&this.benchmark, "bench", false, "") cmdFlags.StringVar(&this.benchmarkMaster, "master", "", "") cmdFlags.BoolVar(&this.pub, "pub", false, "") cmdFlags.BoolVar(&this.sub, "sub", false, "") cmdFlags.BoolVar(&this.benchmarkAsync, "async", false, "") cmdFlags.BoolVar(&this.curl, "curl", false, "") if err := cmdFlags.Parse(args); err != nil { return 2 } if this.benchmark { if validateArgs(this, this.Ui). require("-z"). requireAdminRights("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) zone := ctx.Zone(zkzone.Name()) this.benchApp = zone.SmokeApp this.benchSecret = zone.SmokeSecret this.benchTopic = zone.SmokeTopic this.benchVer = zone.SmokeTopicVersion this.benchPubEndpoint = zone.PubEndpoint if this.id != "" { kws, err := zkzone.KatewayInfos() swallow(err) for _, kw := range kws { if kw.Id == this.id { this.benchPubEndpoint = kw.PubAddr break } } } this.benchId = fmt.Sprintf("%s-%s", ctx.Hostname(), strings.Replace(uuid.New(), "-", "", -1)) this.runBenchmark(zkzone) return } if this.flameGraph { if validateArgs(this, this.Ui). require("-z", "-id"). requireAdminRights("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.generateFlameGraph(zkzone) return } if this.visualLog != "" { this.doVisualize() return } if this.pub { zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.runPub(zkzone) return } if this.sub { zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.runSub(zkzone) return } if this.install { if validateArgs(this, this.Ui). require("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.installGuide(zkzone) return } if this.configOption != "" { this.configMode = true } if this.configMode { if validateArgs(this, this.Ui). require("-z"). requireAdminRights("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) if this.logLevel != "" { if this.id != "" { kw := zkzone.KatewayInfoById(this.id) if kw == nil { panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id)) } this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel)) } else { // apply on all kateways kws, _ := zkzone.KatewayInfos() for _, kw := range kws { this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel)) } } } if this.configOption != "" { parts := strings.SplitN(this.configOption, "=", 2) if len(parts) != 2 { this.Ui.Error("usage: key=value") return } k, v := parts[0], parts[1] if this.id != "" { kw := zkzone.KatewayInfoById(this.id) if kw == nil { panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id)) } this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v)) } else { // apply on all kateways kws, _ := zkzone.KatewayInfos() for _, kw := range kws { this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v)) } } } return } if this.checkup { if this.zone == "" { forAllSortedZones(func(zkzone *zk.ZkZone) { this.runCheckup(zkzone) }) return } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.runCheckup(zkzone) return } if this.showZkNodes { this.Ui.Output(fmt.Sprintf(`%s pubsub manager db dsn %s job db cluster config %s turn off webhook dir`, color.Green("%-50s", zk.KatewayMysqlPath), color.Green("%-50s", zk.PubsubJobConfig), color.Green("%-50s", zk.PubsubWebhooksOff))) return } // display mode lines := make([]string, 0) header := "Zone|Id|Ip|Pprof|Build|Cpu|Heap|Obj|Go|P/S|hhIn/hhOut|Uptime" lines = append(lines, header) forSortedZones(func(zkzone *zk.ZkZone) { if this.zone != "" && zkzone.Name() != this.zone { return } if !this.versionOnly { mysqlDsn, err := zkzone.KatewayMysqlDsn() if err != nil { this.Ui.Warn(fmt.Sprintf("kateway[%s] mysql DSN not set on zk yet", zkzone.Name())) this.Ui.Output(fmt.Sprintf("e,g. %s -> pubsub:pubsub@tcp(10.77.135.217:10010)/pubsub?charset=utf8&timeout=10s", zk.KatewayMysqlPath)) } else { this.Ui.Output(fmt.Sprintf("zone[%s] manager db: %s", color.Cyan(zkzone.Name()), mysqlDsn)) } } kateways, err := zkzone.KatewayInfos() if err != nil { if err == zklib.ErrNoNode { this.Ui.Output("no kateway running") return } else { swallow(err) } } for _, kw := range kateways { if this.id != "" && this.id != kw.Id { continue } statusMap, _ := this.getKatewayStatusMap(kw.ManAddr) logLevel, _ := statusMap["loglevel"].(string) heapSize, _ := statusMap["heap"].(string) heapObjs, _ := statusMap["objects"].(string) pubConn, _ := statusMap["pubconn"].(string) hhAppendN, _ := statusMap["hh_appends"].(string) hhDeliverN, _ := statusMap["hh_delivers"].(string) subConn, _ := statusMap["subconn"].(string) goN, _ := statusMap["goroutines"].(string) if this.versionOnly { pprofAddr := kw.DebugAddr if len(pprofAddr) > 0 && pprofAddr[0] == ':' { pprofAddr = kw.Ip + pprofAddr } pprofAddr = fmt.Sprintf("%s/debug/pprof/", pprofAddr) lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s/%s|%s|%s|%s|%s|%s/%s|%s/%s|%s", zkzone.Name(), kw.Id, kw.Ip, pprofAddr, kw.Build, kw.BuiltAt, kw.Cpu, heapSize, heapObjs, goN, pubConn, subConn, hhAppendN, hhDeliverN, gofmt.PrettySince(kw.Ctime))) continue } this.Ui.Info(fmt.Sprintf("id:%-2s host:%s cpu:%-2s up:%s", kw.Id, kw.Host, kw.Cpu, gofmt.PrettySince(kw.Ctime))) this.Ui.Output(fmt.Sprintf(" ver: %s\n arch: %s\n build: %s\n built: %s\n log: %s\n pub: %s\n sub: %s\n man: %s\n dbg: %s", kw.Ver, kw.Arch, color.Red(kw.Build), kw.BuiltAt, logLevel, kw.PubAddr, kw.SubAddr, kw.ManAddr, kw.DebugAddr, )) if this.longFmt { this.Ui.Output(" full status:") this.Ui.Output(this.getKatewayStatus(kw.ManAddr)) } } }) if this.versionOnly && len(lines) > 1 { fmt.Println(columnize.SimpleFormat(lines)) } return }
func (this *Kateway) runCheckup(zkzone *zk.ZkZone) { zone := ctx.Zone(zkzone.Name()) var ( myApp = zone.SmokeApp hisApp = zone.SmokeHisApp secret = zone.SmokeSecret ver = zone.SmokeTopicVersion topic = zone.SmokeTopic group = zone.SmokeGroup ) if myApp == "" || secret == "" { this.Ui.Warn(fmt.Sprintf("zone[%s] skipped", zkzone.Name())) return } rand.Seed(time.Now().UTC().UnixNano()) kws, err := zkzone.KatewayInfos() swallow(err) if zone.PubEndpoint != "" && zone.SubEndpoint != "" { // add the load balancer endpoint kws = append(kws, &zk.KatewayMeta{ Id: "loadbalancer", PubAddr: zone.PubEndpoint, SubAddr: zone.SubEndpoint, }) } for _, kw := range kws { if this.id != "" && kw.Id != this.id { continue } this.Ui.Info(fmt.Sprintf("zone[%s] kateway[%s]", zkzone.Name(), kw.Id)) // pub a message cf := api.DefaultConfig(myApp, secret) cf.Pub.Endpoint = kw.PubAddr cf.Sub.Endpoint = kw.SubAddr cli := api.NewClient(cf) pubMsg := fmt.Sprintf("gk smoke test msg: [%s]", time.Now()) if this.curl { this.Ui.Output(fmt.Sprintf(`curl -XPOST -H'Appid: %s' -H'Pubkey: %s' -d '%s' %s`, myApp, secret, pubMsg, fmt.Sprintf("http://%s/v1/msgs/%s/%s", kw.PubAddr, topic, ver))) } err = cli.Pub("", []byte(pubMsg), api.PubOption{ Topic: topic, Ver: ver, }) swallow(err) if this.curl { this.Ui.Output(fmt.Sprintf(`curl -XGET -H'Appid: %s' -H'Subkey: %s' %s`, myApp, secret, fmt.Sprintf("http://%s/v1/msgs/%s/%s/%s?group=%s", kw.SubAddr, hisApp, topic, ver, group))) } // confirm that sub can get the pub'ed message err = cli.Sub(api.SubOption{ AppId: hisApp, Topic: topic, Ver: ver, Group: group, AutoClose: true, }, func(statusCode int, subMsg []byte) error { if statusCode != http.StatusOK { return fmt.Errorf("unexpected http status: %s, body:%s", http.StatusText(statusCode), string(subMsg)) } if len(subMsg) < 10 { this.Ui.Warn(fmt.Sprintf("unexpected sub msg: %s", string(subMsg))) } return api.ErrSubStop }) swallow(err) this.Ui.Info(fmt.Sprintf(" ok for %s@%s", kw.Id, kw.Build)) // wait for server cleanup the sub conn time.Sleep(time.Second) // 1. 查询某个pubsub topic的partition数量 // 2. 查看pubsub系统某个topic的生产、消费状态 // 3. pub // 4. sub } }
func getConnectedAdmin(zone string) helix.HelixAdmin { adm := zk.NewZkHelixAdmin(ctx.Zone(zone).ZkHelix) must(adm.Connect()) return adm }
func getConnectedManager(zone, cluster string, it helix.InstanceType) helix.HelixManager { m, err := zk.NewZkHelixManager(cluster, "localhost", "10009", ctx.Zone(zone).ZkHelix, it) must(err) must(m.Connect()) return m }
func (this *Deploy) demo() { var ( maxPort int myPort = -1 myBrokerId = -1 ) this.zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) { maxBrokerId := -1 for _, broker := range liveBrokers { if maxPort < broker.Port { maxPort = broker.Port } if cluster == this.cluster { myPort = broker.Port bid, _ := strconv.Atoi(broker.Id) if bid > maxBrokerId { maxBrokerId = bid } myBrokerId = maxBrokerId + 1 // next deployable broker id } } }) ip, err := ctx.LocalIP() swallow(err) if myPort == -1 { // the 1st deployment of this cluster myPort = maxPort + 1 } if myBrokerId == -1 { // 1st broker id starts with 0 myBrokerId = 0 } logDirs := make([]string, 0) for i := 0; i <= 15; i++ { logDir := fmt.Sprintf("/data%d/%s", i, this.clusterName()) if gio.DirExists(filepath.Dir(logDir)) { logDirs = append(logDirs, logDir) } } if len(logDirs) == 0 { // deploy on a small disk host, having no /dataX dirs logDirs = []string{fmt.Sprintf("%s/logs", this.instanceDir())} } influxAddr := ctx.Zone(this.zone).InfluxAddr if influxAddr != "" { this.Ui.Output(fmt.Sprintf("gk deploy -z %s -c %s -broker.id %d -port %d -ip %s -log.dirs %s -influx %s", this.zone, this.cluster, myBrokerId, myPort, ip.String(), strings.Join(logDirs, ","), influxAddr)) } else { this.Ui.Output(fmt.Sprintf("gk deploy -z %s -c %s -broker.id %d -port %d -ip %s -log.dirs %s", this.zone, this.cluster, myBrokerId, myPort, ip.String(), strings.Join(logDirs, ","))) } }
func (this *Zones) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("zones", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.BoolVar(&this.ipInNumber, "n", false, "") cmdFlags.BoolVar(&this.plain, "plain", false, "") cmdFlags.BoolVar(&this.longFmt, "l", true, "") cmdFlags.BoolVar(&this.nameOnly, "s", false, "") cmdFlags.StringVar(&this.zone, "z", "", "") cmdFlags.BoolVar(&this.influxOnly, "i", false, "") if err := cmdFlags.Parse(args); err != nil { return 2 } // print all by default lines := make([]string, 0) var header string if this.longFmt { header = "Zone|ZkAddr|InfluxDB" } else { header = "Zone|ZkAddr" } lines = append(lines, header) zones := make([][]string, 0) defaultZone := ctx.ZkDefaultZone() for _, zone := range ctx.SortedZones() { if !patternMatched(zone, this.zone) { continue } influxDbAddr := ctx.Zone(zone).InfluxAddr if this.influxOnly && influxDbAddr == "" { continue } if defaultZone == zone { if this.ipInNumber { if this.longFmt { lines = append(lines, fmt.Sprintf("%s*|%s|%s", zone, ctx.ZoneZkAddrs(zone), influxDbAddr)) } else { lines = append(lines, fmt.Sprintf("%s*|%s", zone, ctx.ZoneZkAddrs(zone))) } zones = append(zones, []string{zone + "*", ctx.ZoneZkAddrs(zone), influxDbAddr}) } else { if this.longFmt { lines = append(lines, fmt.Sprintf("%s*|%s|%s", zone, ctx.NamedZoneZkAddrs(zone), influxDbAddr)) } else { lines = append(lines, fmt.Sprintf("%s*|%s", zone, ctx.NamedZoneZkAddrs(zone))) } zones = append(zones, []string{zone + "*", ctx.NamedZoneZkAddrs(zone), influxDbAddr}) } continue } if this.ipInNumber { if this.longFmt { lines = append(lines, fmt.Sprintf("%s|%s|%s", zone, ctx.ZoneZkAddrs(zone), influxDbAddr)) } else { lines = append(lines, fmt.Sprintf("%s|%s", zone, ctx.ZoneZkAddrs(zone))) } zones = append(zones, []string{zone, ctx.ZoneZkAddrs(zone), influxDbAddr}) } else { if this.longFmt { lines = append(lines, fmt.Sprintf("%s|%s|%s", zone, ctx.NamedZoneZkAddrs(zone), influxDbAddr)) } else { lines = append(lines, fmt.Sprintf("%s|%s", zone, ctx.NamedZoneZkAddrs(zone))) } zones = append(zones, []string{zone, ctx.NamedZoneZkAddrs(zone), influxDbAddr}) } } if this.nameOnly { for _, z := range zones { // default zone name ends with '*' fmt.Println(strings.TrimRight(z[0], "*")) } return } if this.plain { for _, z := range zones { this.Ui.Output(fmt.Sprintf("%s:", z[0])) this.Ui.Output(fmt.Sprintf("%s", z[1])) if len(z) > 2 && len(z[2]) > 0 { this.Ui.Output(fmt.Sprintf("influxdb: %s", z[2])) } this.Ui.Output("") } return } if len(lines) > 1 { this.Ui.Output(columnize.SimpleFormat(lines)) } return }
func main() { ctx.LoadFromHome() setupLogging() app := os.Args[0] args := os.Args[1:] for _, arg := range args { if arg == "-v" || arg == "--version" { newArgs := make([]string, len(args)+1) newArgs[0] = "version" copy(newArgs[1:], args) args = newArgs break } if arg == "--generate-bash-completion" { if len(args) > 1 { // contextual auto complete lastArg := args[len(args)-2] switch lastArg { case "-z": // zone for _, zone := range ctx.SortedZones() { if ctx.Zone(zone).ZkHelix != "" { fmt.Println(zone) } } return case "-c": // cluster zone := ctx.ZkDefaultZone() for i := 0; i < len(args)-1; i++ { if args[i] == "-z" { zone = args[i+1] } } adm := zk.NewZkHelixAdmin(ctx.Zone(zone).ZkHelix) if err := adm.Connect(); err != nil { panic(err) } clusters, err := adm.Clusters() if err != nil { panic(err) } for _, cluster := range clusters { fmt.Println(cluster) } return } } for name := range commands { fmt.Println(name) } return } } c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt) c.Args = os.Args[1:] if len(os.Args) > 1 { // command given, convert alias if alias, present := ctx.Alias(os.Args[1]); present { var cargs []string cargs = append(cargs, strings.Split(alias, " ")...) if len(os.Args) > 2 { cargs = append(cargs, os.Args[2:]...) } c.Args = cargs } } c.Commands = commands c.HelpFunc = func(m map[string]cli.CommandFactory) string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("Helix administration console\n\n")) buf.WriteString(cli.BasicHelpFunc(app)(m)) return buf.String() } exitCode, err := c.Run() if err != nil { fmt.Fprintf(os.Stderr, "%+v\n", err) os.Exit(1) } else if c.IsVersion() { os.Exit(0) } os.Exit(exitCode) }