func newGatewayForTest(b *testing.B, store string) *Gateway { zone := os.Getenv("BENCH_ZONE") if zone == "" { zone = "local" } Options.Zone = zone Options.PubHttpAddr = ":9191" Options.SubHttpAddr = ":9192" Options.ManHttpAddr = ":9193" Options.Store = store Options.PubPoolCapcity = 100 Options.Debug = false Options.ManagerStore = "dummy" Options.DummyCluster = "me" Options.MaxPubSize = 1 << 20 Options.MetaRefresh = time.Hour Options.ReporterInterval = time.Hour Options.InfluxServer = "none" Options.InfluxDbName = "none" Options.JobStore = "dummy" Options.EnableHintedHandoff = true Options.HintedHandoffDir = "hhdata" Options.HintedHandoffType = "disk" os.RemoveAll("hhdata") ctx.LoadFromHome() gw := New("1") if err := gw.Start(); err != nil { b.Fatal(err) } return gw }
func init() { flag.StringVar(&Options.Zone, "z", "", "zone") flag.BoolVar(&Options.ShowVersion, "v", false, "show version and exit") flag.BoolVar(&Options.ShowVersion, "version", false, "show version and exit") flag.StringVar(&Options.LogFile, "log", "stdout", "log file") flag.StringVar(&Options.LogLevel, "level", "debug", "log level") flag.IntVar(&Options.LogRotateSize, "logsize", 10<<30, "max unrotated log file size") flag.StringVar(&Options.InfluxAddr, "influxaddr", "", "influxdb server addr") flag.StringVar(&Options.ManagerType, "man", "dummy", "manager type <dummy|mysql>") flag.StringVar(&Options.InfluxDbname, "influxdb", "", "influxdb db name") flag.StringVar(&Options.ListenAddr, "addr", ":9065", "monitor http server addr") flag.StringVar(&Options.HintedHandoffDir, "hhdirs", "hh", "hinted handoff dirs seperated by comma") flag.Parse() if Options.ShowVersion { fmt.Fprintf(os.Stderr, "%s-%s\n", gafka.Version, gafka.BuildId) os.Exit(0) } if Options.Zone == "" { panic("empty zone not allowed") } golog.SetOutput(ioutil.Discard) if Options.LogFile != "stdout" { SetupLogging(Options.LogFile, Options.LogLevel, "panic") } else { SetupLogging(Options.LogFile, Options.LogLevel, "") } ctx.LoadFromHome() }
func main() { ctx.LoadFromHome() log4go.AddFilter("stdout", log4go.INFO, log4go.NewConsoleLogWriter()) app := os.Args[0] args := os.Args[1:] for _, arg := range args { if arg == "-v" || arg == "--version" { newArgs := make([]string, len(args)+1) newArgs[0] = "version" copy(newArgs[1:], args) args = newArgs break } } c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId) c.Args = os.Args[1:] c.Commands = commands c.HelpFunc = func(m map[string]cli.CommandFactory) string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("A MySQL syncer tool\n\n")) buf.WriteString(cli.BasicHelpFunc(app)(m)) return buf.String() } exitCode, err := c.Run() if err != nil { fmt.Fprintf(os.Stderr, "%+v\n", err) os.Exit(1) } else if c.IsVersion() { os.Exit(0) } os.Exit(exitCode) }
// 145 ns/op 16 B/op 1 allocs/op func BenchmarkKafkaTopicObfuscationWithMpool(b *testing.B) { ctx.LoadFromHome() m := New(DefaultConfig("local")) for i := 0; i < b.N; i++ { m.KafkaTopic("appid", "topic", "v10") } }
func main() { ctx.LoadFromHome() setupLogging() app := os.Args[0] args := os.Args[1:] for _, arg := range args { if arg == "-v" || arg == "--version" { newArgs := make([]string, len(args)+1) newArgs[0] = "version" copy(newArgs[1:], args) args = newArgs break } if arg == "--generate-bash-completion" { for name := range commands { fmt.Println(name) } for _, cmd := range ctx.Aliases() { fmt.Println(cmd) } return } } c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt) c.Args = os.Args[1:] if len(os.Args) > 1 { // command given, convert alias if alias, present := ctx.Alias(os.Args[1]); present { cargs := make([]string, 0) cargs = append(cargs, strings.Split(alias, " ")...) if len(os.Args) > 2 { cargs = append(cargs, os.Args[2:]...) } c.Args = cargs } } c.Commands = commands c.HelpFunc = func(m map[string]cli.CommandFactory) string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("Unified multi-datacenter multi-cluster kafka swiss-knife management console\n\n")) buf.WriteString(cli.BasicHelpFunc(app)(m)) return buf.String() } exitCode, err := c.Run() if err != nil { fmt.Fprintf(os.Stderr, "%+v\n", err) os.Exit(1) } else if c.IsVersion() { os.Exit(0) } os.Exit(exitCode) }
func TestKafkaTopicWithObfuscation(t *testing.T) { ctx.LoadFromHome() m := New(DefaultConfig("local")) appid := "app1" topic := "foobar" ver := "v10" m.appSecretMap = make(map[string]string) m.appSecretMap[appid] = "b7b73ac504d84944a3fedb801b348b2e" t.Logf("topic: %s", m.KafkaTopic(appid, topic, ver)) assert.Equal(t, "app1.foobar.v10.844", m.KafkaTopic(appid, topic, ver)) }
func New(zone, cluster string) *KafkaFS { log.Info("starting kfs zone:%s, cluster:%s", zone, cluster) this := &KafkaFS{} ctx.LoadFromHome() zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) this.zkcluster = zkzone.NewCluster(cluster) // panic if invalid cluster this.root = this.newDir(os.FileMode(0777)) return this }
func (this *Start) main() { ctx.LoadFromHome() this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) zkConnEvt, ok := this.zkzone.SessionEvents() if !ok { panic("someone stealing my events") } registry.Default = zkr.New(this.zkzone) log.Info("ehaproxy[%s] starting...", gafka.BuildId) go this.runMonitorServer(this.httpAddr) zkConnected := false for { instances, instancesChange, err := registry.Default.WatchInstances() if err != nil { log.Error("zone[%s] %s", this.zkzone.Name(), err) time.Sleep(time.Second) continue } if zkConnected { if len(instances) > 0 { this.reload(instances) } else { // resilience to zk problem by local cache log.Warn("backend all shutdown? skip this change") time.Sleep(time.Second) continue } } select { case <-this.quitCh: return case evt := <-zkConnEvt: if evt.State == zklib.StateHasSession && !zkConnected { log.Info("zk connected") zkConnected = true } else if zkConnected { log.Warn("zk jitter: %+v", evt) } case <-instancesChange: log.Info("instances changed!!") } } }
func newGatewayForTest(b *testing.B, store string) *Gateway { Options.Zone = "local" Options.PubHttpAddr = ":9191" Options.SubHttpAddr = ":9192" Options.Store = store Options.Debug = false Options.DisableMetrics = false ctx.LoadFromHome() gw := New("1") if err := gw.Start(); err != nil { b.Fatal(err) } return gw }
func newGatewayForTest(b *testing.B, store string) *Gateway { options.zone = "local" options.pubHttpAddr = ":9191" options.subHttpAddr = ":9192" options.store = store options.debug = false options.disableMetrics = false options.enableBreaker = true ctx.LoadFromHome() gw := New("1") if err := gw.Start(); err != nil { b.Fatal(err) } return gw }
func BenchmarkDirectKafkaProduce1K(b *testing.B) { msgSize := 1 << 10 b.ReportAllocs() b.SetBytes(int64(msgSize)) ctx.LoadFromHome() cf := zkmeta.DefaultConfig() zkzone := zk.NewZkZone(zk.DefaultConfig("local", ctx.ZoneZkAddrs("local"))) cf.Refresh = time.Hour meta.Default = zkmeta.New(cf, zkzone) meta.Default.Start() store.DefaultPubStore = kafka.NewPubStore(100, 0, false, false, true) store.DefaultPubStore.Start() data := []byte(strings.Repeat("X", msgSize)) for i := 0; i < b.N; i++ { store.DefaultPubStore.SyncPub("me", "foobar", nil, data) } }
func (this *Monitor) Init() { var logFile, zone string flag.StringVar(&logFile, "log", "stdout", "log filename") flag.StringVar(&zone, "z", "", "zone, required") flag.StringVar(&this.apiAddr, "http", ":10025", "api http server addr") flag.StringVar(&this.influxdbAddr, "influxAddr", "", "influxdb addr, required") flag.StringVar(&this.influxdbDbName, "db", "", "influxdb db name, required") flag.StringVar(&this.externalDir, "confd", "", "external script config dir") flag.Parse() if zone == "" || this.influxdbDbName == "" || this.influxdbAddr == "" { panic("zone or influxdb empty, run help ") } ctx.LoadFromHome() this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) this.watchers = make([]Watcher, 0, 10) this.quit = make(chan struct{}) // export RESTful api this.setupRoutes() if logFile == "stdout" { log.AddFilter("stdout", log.TRACE, log.NewConsoleLogWriter()) } else { log.DeleteFilter("stdout") filer := log.NewFileLogWriter(logFile, true, false, 0644) filer.SetRotateDaily(true) filer.SetFormat("[%d %T] [%L] (%S) %M") log.AddFilter("file", log.TRACE, filer) } rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute) if err != nil { panic(err) } telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc) }
func (this *Start) main() { ctx.LoadFromHome() // TODO zk session timeout err := etclib.Dial(strings.Split(ctx.ZoneZkAddrs(this.zone), ",")) swalllow(err) root := zkr.Root(this.zone) ch := make(chan []string, 10) go etclib.WatchChildren(root, ch) var servers = BackendServers{ CpuNum: ctx.NumCPU(), HaproxyRoot: this.root, ForwardFor: this.forwardFor, PubPort: this.pubPort, SubPort: this.subPort, ManPort: this.manPort, } var lastInstances []string for { select { case <-this.quitCh: time.Sleep(time.Second) // FIXME just wait log flush return case <-ch: kwInstances, err := etclib.Children(root) if err != nil { log.Error("%s: %v", root, err) continue } log.Info("kateway ids: %+v -> %+v", lastInstances, kwInstances) lastInstances = kwInstances servers.reset() for _, kwId := range kwInstances { kwNode := fmt.Sprintf("%s/%s", root, kwId) data, err := etclib.Get(kwNode) if err != nil { log.Error("%s: %v", kwNode, err) continue } info := make(map[string]string) if err = json.Unmarshal([]byte(data), &info); err != nil { log.Error("%s: %v", data, err) continue } // pub if info["pub"] != "" { be := Backend{ Name: "p" + info["id"], Addr: info["pub"], Cpu: info["cpu"], } servers.Pub = append(servers.Pub, be) } // sub if info["sub"] != "" { be := Backend{ Name: "s" + info["id"], Addr: info["sub"], Cpu: info["cpu"], } servers.Sub = append(servers.Sub, be) } // man if info["man"] != "" { be := Backend{ Name: "m" + info["id"], Addr: info["man"], Cpu: info["cpu"], } servers.Man = append(servers.Man, be) } } if err = this.createConfigFile(servers); err != nil { log.Error(err) continue } if err = this.reloadHAproxy(); err != nil { log.Error("reloading haproxy: %v", err) panic(err) } } } }
func main() { defer func() { if err := recover(); err != nil { fmt.Println(err) debug.PrintStack() } }() if gateway.Options.Debug { log.SetFlags(log.LstdFlags | log.Llongfile) // TODO zk sdk uses this log.SetPrefix(color.Magenta("[log]")) } else { log.SetOutput(ioutil.Discard) } if gateway.Options.KillFile != "" { if err := signal.SignalProcessByPidFile(gateway.Options.KillFile, syscall.SIGUSR2); err != nil { panic(err) } fmt.Println("kateway killed") os.Exit(0) } if gateway.Options.GolangTrace { // go tool trace kateway xxx.pprof f, err := os.Create(time.Now().Format("2006-01-02T150405.pprof")) if err != nil { panic(err) } defer f.Close() if err = trace.Start(f); err != nil { panic(err) } defer trace.Stop() } fmt.Fprintln(os.Stderr, strings.TrimSpace(logo)) if gateway.Options.PidFile != "" { pid := os.Getpid() if err := ioutil.WriteFile(gateway.Options.PidFile, []byte(fmt.Sprintf("%d", pid)), 0644); err != nil { panic(err) } } gateway.SetupLogging(gateway.Options.LogFile, gateway.Options.LogLevel, gateway.Options.CrashLogFile) // load config if gateway.Options.ConfigFile == "" { ctx.LoadFromHome() } else { _, err := os.Stat(gateway.Options.ConfigFile) if err != nil { panic(err) } ctx.LoadConfig(gateway.Options.ConfigFile) } gw := gateway.New(gateway.Options.Id) if err := gw.Start(); err != nil { glog.Error(err.Error()) glog.Info("kateway[%s@%s] bye!", gafka.BuildId, gafka.BuiltAt) glog.Close() os.Exit(1) } t0 := time.Now() gw.ServeForever() glog.Info("kateway[%s@%s] %s, bye!", gafka.BuildId, gafka.BuiltAt, time.Since(t0)) glog.Close() if gateway.Options.PidFile != "" { syscall.Unlink(gateway.Options.PidFile) } }
func createRunner() *runner { ctx.LoadFromHome() cf, _ := NewConfig("uri", "db", "user", "pass", time.Second) return New(metrics.DefaultRegistry, cf).(*runner) }
func init() { ctx.LoadFromHome() log.Disable() }
func init() { ctx.LoadFromHome() }
func init() { ctx.LoadFromHome() log.AddFilter("stdout", log.INFO, log.NewConsoleLogWriter()) }
func main() { ctx.LoadFromHome() setupLogging() app := os.Args[0] args := os.Args[1:] for _, arg := range args { if arg == "-v" || arg == "--version" { newArgs := make([]string, len(args)+1) newArgs[0] = "version" copy(newArgs[1:], args) args = newArgs break } if arg == "--generate-bash-completion" { if len(args) > 1 { // contextual auto complete lastArg := args[len(args)-2] switch lastArg { case "-z": // zone for _, zone := range ctx.SortedZones() { if ctx.Zone(zone).ZkHelix != "" { fmt.Println(zone) } } return case "-c": // cluster zone := ctx.ZkDefaultZone() for i := 0; i < len(args)-1; i++ { if args[i] == "-z" { zone = args[i+1] } } adm := zk.NewZkHelixAdmin(ctx.Zone(zone).ZkHelix) if err := adm.Connect(); err != nil { panic(err) } clusters, err := adm.Clusters() if err != nil { panic(err) } for _, cluster := range clusters { fmt.Println(cluster) } return } } for name := range commands { fmt.Println(name) } return } } c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt) c.Args = os.Args[1:] if len(os.Args) > 1 { // command given, convert alias if alias, present := ctx.Alias(os.Args[1]); present { var cargs []string cargs = append(cargs, strings.Split(alias, " ")...) if len(os.Args) > 2 { cargs = append(cargs, os.Args[2:]...) } c.Args = cargs } } c.Commands = commands c.HelpFunc = func(m map[string]cli.CommandFactory) string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("Helix administration console\n\n")) buf.WriteString(cli.BasicHelpFunc(app)(m)) return buf.String() } exitCode, err := c.Run() if err != nil { fmt.Fprintf(os.Stderr, "%+v\n", err) os.Exit(1) } else if c.IsVersion() { os.Exit(0) } os.Exit(exitCode) }
func main() { ctx.LoadFromHome() setupLogging() app := os.Args[0] args := os.Args[1:] for _, arg := range args { if arg == "-v" || arg == "--version" { newArgs := make([]string, len(args)+1) newArgs[0] = "version" copy(newArgs[1:], args) args = newArgs break } if arg == "--generate-bash-completion" { if len(args) > 1 { // contextual auto complete lastArg := args[len(args)-2] switch lastArg { case "-z": // zone for _, zone := range ctx.SortedZones() { fmt.Println(zone) } return case "-c": // cluster zone := ctx.ZkDefaultZone() for i := 0; i < len(args)-1; i++ { if args[i] == "-z" { zone = args[i+1] } } zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { fmt.Println(zkcluster.Name()) }) return case "-t": // topic zone := ctx.ZkDefaultZone() var cluster string for i := 0; i < len(args)-1; i++ { if args[i] == "-z" { zone = args[i+1] } if args[i] == "-c" { cluster = args[i+1] } } if cluster != "" { zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) zkcluster := zkzone.NewCluster(cluster) topics, _ := zkcluster.Topics() for _, t := range topics { fmt.Println(t) } } return } } for name := range commands { fmt.Println(name) } for _, cmd := range ctx.Aliases() { fmt.Println(cmd) } return } } c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt) c.Args = os.Args[1:] if len(os.Args) > 1 { // command given, convert alias if alias, present := ctx.Alias(os.Args[1]); present { var cargs []string cargs = append(cargs, strings.Split(alias, " ")...) if len(os.Args) > 2 { cargs = append(cargs, os.Args[2:]...) } c.Args = cargs } } c.Commands = commands c.HelpFunc = func(m map[string]cli.CommandFactory) string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("Unified multi-datacenter multi-cluster kafka swiss-knife management console\n\n")) buf.WriteString(cli.BasicHelpFunc(app)(m)) return buf.String() } exitCode, err := c.Run() if err != nil { fmt.Fprintf(os.Stderr, "%+v\n", err) os.Exit(1) } else if c.IsVersion() { os.Exit(0) } os.Exit(exitCode) }
func main() { ctx.LoadFromHome() log.SetOutput(ioutil.Discard) log4go.AddFilter("stdout", log4go.INFO, log4go.NewConsoleLogWriter()) app := os.Args[0] args := os.Args[1:] for _, arg := range args { if arg == "-v" || arg == "--version" { newArgs := make([]string, len(args)+1) newArgs[0] = "version" copy(newArgs[1:], args) args = newArgs break } if arg == "--generate-bash-completion" { if len(args) > 1 { // contextual auto complete lastArg := args[len(args)-2] switch lastArg { case "-z": // zone for _, zone := range ctx.SortedZones() { fmt.Println(zone) } return } } for name, _ := range commands { fmt.Println(name) } return } } c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId) c.Args = os.Args[1:] if len(os.Args) > 1 { // command given, convert alias if alias, present := ctx.Alias(os.Args[1]); present { var cargs []string cargs = append(cargs, strings.Split(alias, " ")...) if len(os.Args) > 2 { cargs = append(cargs, os.Args[2:]...) } c.Args = cargs } } c.Commands = commands c.HelpFunc = func(m map[string]cli.CommandFactory) string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("A CLI tool for Zookeeper\n\n")) buf.WriteString(cli.BasicHelpFunc(app)(m)) return buf.String() } exitCode, err := c.Run() if err != nil { fmt.Fprintf(os.Stderr, "%+v\n", err) os.Exit(1) } else if c.IsVersion() { os.Exit(0) } os.Exit(exitCode) }