func newGatewayForTest(b *testing.B, store string) *Gateway {
zone := os.Getenv("BENCH_ZONE")
if zone == "" {
zone = "local"
}
Options.Zone = zone
Options.PubHttpAddr = ":9191"
Options.SubHttpAddr = ":9192"
Options.ManHttpAddr = ":9193"
Options.Store = store
Options.PubPoolCapcity = 100
Options.Debug = false
Options.ManagerStore = "dummy"
Options.DummyCluster = "me"
Options.MaxPubSize = 1 << 20
Options.MetaRefresh = time.Hour
Options.ReporterInterval = time.Hour
Options.InfluxServer = "none"
Options.InfluxDbName = "none"
Options.JobStore = "dummy"
Options.EnableHintedHandoff = true
Options.HintedHandoffDir = "hhdata"
Options.HintedHandoffType = "disk"
os.RemoveAll("hhdata")
ctx.LoadFromHome()
gw := New("1")
if err := gw.Start(); err != nil {
b.Fatal(err)
}
return gw
}
func init() {
flag.StringVar(&Options.Zone, "z", "", "zone")
flag.BoolVar(&Options.ShowVersion, "v", false, "show version and exit")
flag.BoolVar(&Options.ShowVersion, "version", false, "show version and exit")
flag.StringVar(&Options.LogFile, "log", "stdout", "log file")
flag.StringVar(&Options.LogLevel, "level", "debug", "log level")
flag.IntVar(&Options.LogRotateSize, "logsize", 10<<30, "max unrotated log file size")
flag.StringVar(&Options.InfluxAddr, "influxaddr", "", "influxdb server addr")
flag.StringVar(&Options.ManagerType, "man", "dummy", "manager type <dummy|mysql>")
flag.StringVar(&Options.InfluxDbname, "influxdb", "", "influxdb db name")
flag.StringVar(&Options.ListenAddr, "addr", ":9065", "monitor http server addr")
flag.StringVar(&Options.HintedHandoffDir, "hhdirs", "hh", "hinted handoff dirs seperated by comma")
flag.Parse()
if Options.ShowVersion {
fmt.Fprintf(os.Stderr, "%s-%s\n", gafka.Version, gafka.BuildId)
os.Exit(0)
}
if Options.Zone == "" {
panic("empty zone not allowed")
}
golog.SetOutput(ioutil.Discard)
if Options.LogFile != "stdout" {
SetupLogging(Options.LogFile, Options.LogLevel, "panic")
} else {
SetupLogging(Options.LogFile, Options.LogLevel, "")
}
ctx.LoadFromHome()
}
func main() {
ctx.LoadFromHome()
log4go.AddFilter("stdout", log4go.INFO, log4go.NewConsoleLogWriter())
app := os.Args[0]
args := os.Args[1:]
for _, arg := range args {
if arg == "-v" || arg == "--version" {
newArgs := make([]string, len(args)+1)
newArgs[0] = "version"
copy(newArgs[1:], args)
args = newArgs
break
}
}
c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId)
c.Args = os.Args[1:]
c.Commands = commands
c.HelpFunc = func(m map[string]cli.CommandFactory) string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("A MySQL syncer tool\n\n"))
buf.WriteString(cli.BasicHelpFunc(app)(m))
return buf.String()
}
exitCode, err := c.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "%+v\n", err)
os.Exit(1)
} else if c.IsVersion() {
os.Exit(0)
}
os.Exit(exitCode)
}
// 145 ns/op 16 B/op 1 allocs/op
func BenchmarkKafkaTopicObfuscationWithMpool(b *testing.B) {
ctx.LoadFromHome()
m := New(DefaultConfig("local"))
for i := 0; i < b.N; i++ {
m.KafkaTopic("appid", "topic", "v10")
}
}
func main() {
ctx.LoadFromHome()
setupLogging()
app := os.Args[0]
args := os.Args[1:]
for _, arg := range args {
if arg == "-v" || arg == "--version" {
newArgs := make([]string, len(args)+1)
newArgs[0] = "version"
copy(newArgs[1:], args)
args = newArgs
break
}
if arg == "--generate-bash-completion" {
for name := range commands {
fmt.Println(name)
}
for _, cmd := range ctx.Aliases() {
fmt.Println(cmd)
}
return
}
}
c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt)
c.Args = os.Args[1:]
if len(os.Args) > 1 {
// command given, convert alias
if alias, present := ctx.Alias(os.Args[1]); present {
cargs := make([]string, 0)
cargs = append(cargs, strings.Split(alias, " ")...)
if len(os.Args) > 2 {
cargs = append(cargs, os.Args[2:]...)
}
c.Args = cargs
}
}
c.Commands = commands
c.HelpFunc = func(m map[string]cli.CommandFactory) string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("Unified multi-datacenter multi-cluster kafka swiss-knife management console\n\n"))
buf.WriteString(cli.BasicHelpFunc(app)(m))
return buf.String()
}
exitCode, err := c.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "%+v\n", err)
os.Exit(1)
} else if c.IsVersion() {
os.Exit(0)
}
os.Exit(exitCode)
}
func TestKafkaTopicWithObfuscation(t *testing.T) {
ctx.LoadFromHome()
m := New(DefaultConfig("local"))
appid := "app1"
topic := "foobar"
ver := "v10"
m.appSecretMap = make(map[string]string)
m.appSecretMap[appid] = "b7b73ac504d84944a3fedb801b348b2e"
t.Logf("topic: %s", m.KafkaTopic(appid, topic, ver))
assert.Equal(t, "app1.foobar.v10.844", m.KafkaTopic(appid, topic, ver))
}
func New(zone, cluster string) *KafkaFS {
log.Info("starting kfs zone:%s, cluster:%s", zone, cluster)
this := &KafkaFS{}
ctx.LoadFromHome()
zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
this.zkcluster = zkzone.NewCluster(cluster) // panic if invalid cluster
this.root = this.newDir(os.FileMode(0777))
return this
}
func (this *Start) main() {
ctx.LoadFromHome()
this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
zkConnEvt, ok := this.zkzone.SessionEvents()
if !ok {
panic("someone stealing my events")
}
registry.Default = zkr.New(this.zkzone)
log.Info("ehaproxy[%s] starting...", gafka.BuildId)
go this.runMonitorServer(this.httpAddr)
zkConnected := false
for {
instances, instancesChange, err := registry.Default.WatchInstances()
if err != nil {
log.Error("zone[%s] %s", this.zkzone.Name(), err)
time.Sleep(time.Second)
continue
}
if zkConnected {
if len(instances) > 0 {
this.reload(instances)
} else {
// resilience to zk problem by local cache
log.Warn("backend all shutdown? skip this change")
time.Sleep(time.Second)
continue
}
}
select {
case <-this.quitCh:
return
case evt := <-zkConnEvt:
if evt.State == zklib.StateHasSession && !zkConnected {
log.Info("zk connected")
zkConnected = true
} else if zkConnected {
log.Warn("zk jitter: %+v", evt)
}
case <-instancesChange:
log.Info("instances changed!!")
}
}
}
func newGatewayForTest(b *testing.B, store string) *Gateway {
Options.Zone = "local"
Options.PubHttpAddr = ":9191"
Options.SubHttpAddr = ":9192"
Options.Store = store
Options.Debug = false
Options.DisableMetrics = false
ctx.LoadFromHome()
gw := New("1")
if err := gw.Start(); err != nil {
b.Fatal(err)
}
return gw
}
func newGatewayForTest(b *testing.B, store string) *Gateway {
options.zone = "local"
options.pubHttpAddr = ":9191"
options.subHttpAddr = ":9192"
options.store = store
options.debug = false
options.disableMetrics = false
options.enableBreaker = true
ctx.LoadFromHome()
gw := New("1")
if err := gw.Start(); err != nil {
b.Fatal(err)
}
return gw
}
func BenchmarkDirectKafkaProduce1K(b *testing.B) {
msgSize := 1 << 10
b.ReportAllocs()
b.SetBytes(int64(msgSize))
ctx.LoadFromHome()
cf := zkmeta.DefaultConfig()
zkzone := zk.NewZkZone(zk.DefaultConfig("local", ctx.ZoneZkAddrs("local")))
cf.Refresh = time.Hour
meta.Default = zkmeta.New(cf, zkzone)
meta.Default.Start()
store.DefaultPubStore = kafka.NewPubStore(100, 0, false, false, true)
store.DefaultPubStore.Start()
data := []byte(strings.Repeat("X", msgSize))
for i := 0; i < b.N; i++ {
store.DefaultPubStore.SyncPub("me", "foobar", nil, data)
}
}
func (this *Monitor) Init() {
var logFile, zone string
flag.StringVar(&logFile, "log", "stdout", "log filename")
flag.StringVar(&zone, "z", "", "zone, required")
flag.StringVar(&this.apiAddr, "http", ":10025", "api http server addr")
flag.StringVar(&this.influxdbAddr, "influxAddr", "", "influxdb addr, required")
flag.StringVar(&this.influxdbDbName, "db", "", "influxdb db name, required")
flag.StringVar(&this.externalDir, "confd", "", "external script config dir")
flag.Parse()
if zone == "" || this.influxdbDbName == "" || this.influxdbAddr == "" {
panic("zone or influxdb empty, run help ")
}
ctx.LoadFromHome()
this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
this.watchers = make([]Watcher, 0, 10)
this.quit = make(chan struct{})
// export RESTful api
this.setupRoutes()
if logFile == "stdout" {
log.AddFilter("stdout", log.TRACE, log.NewConsoleLogWriter())
} else {
log.DeleteFilter("stdout")
filer := log.NewFileLogWriter(logFile, true, false, 0644)
filer.SetRotateDaily(true)
filer.SetFormat("[%d %T] [%L] (%S) %M")
log.AddFilter("file", log.TRACE, filer)
}
rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute)
if err != nil {
panic(err)
}
telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
}
func (this *Start) main() {
ctx.LoadFromHome()
// TODO zk session timeout
err := etclib.Dial(strings.Split(ctx.ZoneZkAddrs(this.zone), ","))
swalllow(err)
root := zkr.Root(this.zone)
ch := make(chan []string, 10)
go etclib.WatchChildren(root, ch)
var servers = BackendServers{
CpuNum: ctx.NumCPU(),
HaproxyRoot: this.root,
ForwardFor: this.forwardFor,
PubPort: this.pubPort,
SubPort: this.subPort,
ManPort: this.manPort,
}
var lastInstances []string
for {
select {
case <-this.quitCh:
time.Sleep(time.Second) // FIXME just wait log flush
return
case <-ch:
kwInstances, err := etclib.Children(root)
if err != nil {
log.Error("%s: %v", root, err)
continue
}
log.Info("kateway ids: %+v -> %+v", lastInstances, kwInstances)
lastInstances = kwInstances
servers.reset()
for _, kwId := range kwInstances {
kwNode := fmt.Sprintf("%s/%s", root, kwId)
data, err := etclib.Get(kwNode)
if err != nil {
log.Error("%s: %v", kwNode, err)
continue
}
info := make(map[string]string)
if err = json.Unmarshal([]byte(data), &info); err != nil {
log.Error("%s: %v", data, err)
continue
}
// pub
if info["pub"] != "" {
be := Backend{
Name: "p" + info["id"],
Addr: info["pub"],
Cpu: info["cpu"],
}
servers.Pub = append(servers.Pub, be)
}
// sub
if info["sub"] != "" {
be := Backend{
Name: "s" + info["id"],
Addr: info["sub"],
Cpu: info["cpu"],
}
servers.Sub = append(servers.Sub, be)
}
// man
if info["man"] != "" {
be := Backend{
Name: "m" + info["id"],
Addr: info["man"],
Cpu: info["cpu"],
}
servers.Man = append(servers.Man, be)
}
}
if err = this.createConfigFile(servers); err != nil {
log.Error(err)
continue
}
if err = this.reloadHAproxy(); err != nil {
log.Error("reloading haproxy: %v", err)
panic(err)
}
}
}
}
func main() {
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
debug.PrintStack()
}
}()
if gateway.Options.Debug {
log.SetFlags(log.LstdFlags | log.Llongfile) // TODO zk sdk uses this
log.SetPrefix(color.Magenta("[log]"))
} else {
log.SetOutput(ioutil.Discard)
}
if gateway.Options.KillFile != "" {
if err := signal.SignalProcessByPidFile(gateway.Options.KillFile, syscall.SIGUSR2); err != nil {
panic(err)
}
fmt.Println("kateway killed")
os.Exit(0)
}
if gateway.Options.GolangTrace {
// go tool trace kateway xxx.pprof
f, err := os.Create(time.Now().Format("2006-01-02T150405.pprof"))
if err != nil {
panic(err)
}
defer f.Close()
if err = trace.Start(f); err != nil {
panic(err)
}
defer trace.Stop()
}
fmt.Fprintln(os.Stderr, strings.TrimSpace(logo))
if gateway.Options.PidFile != "" {
pid := os.Getpid()
if err := ioutil.WriteFile(gateway.Options.PidFile, []byte(fmt.Sprintf("%d", pid)), 0644); err != nil {
panic(err)
}
}
gateway.SetupLogging(gateway.Options.LogFile, gateway.Options.LogLevel, gateway.Options.CrashLogFile)
// load config
if gateway.Options.ConfigFile == "" {
ctx.LoadFromHome()
} else {
_, err := os.Stat(gateway.Options.ConfigFile)
if err != nil {
panic(err)
}
ctx.LoadConfig(gateway.Options.ConfigFile)
}
gw := gateway.New(gateway.Options.Id)
if err := gw.Start(); err != nil {
glog.Error(err.Error())
glog.Info("kateway[%s@%s] bye!", gafka.BuildId, gafka.BuiltAt)
glog.Close()
os.Exit(1)
}
t0 := time.Now()
gw.ServeForever()
glog.Info("kateway[%s@%s] %s, bye!", gafka.BuildId, gafka.BuiltAt, time.Since(t0))
glog.Close()
if gateway.Options.PidFile != "" {
syscall.Unlink(gateway.Options.PidFile)
}
}
func createRunner() *runner {
ctx.LoadFromHome()
cf, _ := NewConfig("uri", "db", "user", "pass", time.Second)
return New(metrics.DefaultRegistry, cf).(*runner)
}
func init() {
ctx.LoadFromHome()
log.Disable()
}
func init() {
ctx.LoadFromHome()
}
func init() {
ctx.LoadFromHome()
log.AddFilter("stdout", log.INFO, log.NewConsoleLogWriter())
}
func main() {
ctx.LoadFromHome()
setupLogging()
app := os.Args[0]
args := os.Args[1:]
for _, arg := range args {
if arg == "-v" || arg == "--version" {
newArgs := make([]string, len(args)+1)
newArgs[0] = "version"
copy(newArgs[1:], args)
args = newArgs
break
}
if arg == "--generate-bash-completion" {
if len(args) > 1 {
// contextual auto complete
lastArg := args[len(args)-2]
switch lastArg {
case "-z": // zone
for _, zone := range ctx.SortedZones() {
if ctx.Zone(zone).ZkHelix != "" {
fmt.Println(zone)
}
}
return
case "-c": // cluster
zone := ctx.ZkDefaultZone()
for i := 0; i < len(args)-1; i++ {
if args[i] == "-z" {
zone = args[i+1]
}
}
adm := zk.NewZkHelixAdmin(ctx.Zone(zone).ZkHelix)
if err := adm.Connect(); err != nil {
panic(err)
}
clusters, err := adm.Clusters()
if err != nil {
panic(err)
}
for _, cluster := range clusters {
fmt.Println(cluster)
}
return
}
}
for name := range commands {
fmt.Println(name)
}
return
}
}
c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt)
c.Args = os.Args[1:]
if len(os.Args) > 1 {
// command given, convert alias
if alias, present := ctx.Alias(os.Args[1]); present {
var cargs []string
cargs = append(cargs, strings.Split(alias, " ")...)
if len(os.Args) > 2 {
cargs = append(cargs, os.Args[2:]...)
}
c.Args = cargs
}
}
c.Commands = commands
c.HelpFunc = func(m map[string]cli.CommandFactory) string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("Helix administration console\n\n"))
buf.WriteString(cli.BasicHelpFunc(app)(m))
return buf.String()
}
exitCode, err := c.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "%+v\n", err)
os.Exit(1)
} else if c.IsVersion() {
os.Exit(0)
}
os.Exit(exitCode)
}
func main() {
ctx.LoadFromHome()
setupLogging()
app := os.Args[0]
args := os.Args[1:]
for _, arg := range args {
if arg == "-v" || arg == "--version" {
newArgs := make([]string, len(args)+1)
newArgs[0] = "version"
copy(newArgs[1:], args)
args = newArgs
break
}
if arg == "--generate-bash-completion" {
if len(args) > 1 {
// contextual auto complete
lastArg := args[len(args)-2]
switch lastArg {
case "-z": // zone
for _, zone := range ctx.SortedZones() {
fmt.Println(zone)
}
return
case "-c": // cluster
zone := ctx.ZkDefaultZone()
for i := 0; i < len(args)-1; i++ {
if args[i] == "-z" {
zone = args[i+1]
}
}
zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
fmt.Println(zkcluster.Name())
})
return
case "-t": // topic
zone := ctx.ZkDefaultZone()
var cluster string
for i := 0; i < len(args)-1; i++ {
if args[i] == "-z" {
zone = args[i+1]
}
if args[i] == "-c" {
cluster = args[i+1]
}
}
if cluster != "" {
zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
zkcluster := zkzone.NewCluster(cluster)
topics, _ := zkcluster.Topics()
for _, t := range topics {
fmt.Println(t)
}
}
return
}
}
for name := range commands {
fmt.Println(name)
}
for _, cmd := range ctx.Aliases() {
fmt.Println(cmd)
}
return
}
}
c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId+"-"+gafka.BuiltAt)
c.Args = os.Args[1:]
if len(os.Args) > 1 {
// command given, convert alias
if alias, present := ctx.Alias(os.Args[1]); present {
var cargs []string
cargs = append(cargs, strings.Split(alias, " ")...)
if len(os.Args) > 2 {
cargs = append(cargs, os.Args[2:]...)
}
c.Args = cargs
}
}
c.Commands = commands
c.HelpFunc = func(m map[string]cli.CommandFactory) string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("Unified multi-datacenter multi-cluster kafka swiss-knife management console\n\n"))
buf.WriteString(cli.BasicHelpFunc(app)(m))
return buf.String()
}
exitCode, err := c.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "%+v\n", err)
os.Exit(1)
} else if c.IsVersion() {
os.Exit(0)
}
os.Exit(exitCode)
}
func main() {
ctx.LoadFromHome()
log.SetOutput(ioutil.Discard)
log4go.AddFilter("stdout", log4go.INFO, log4go.NewConsoleLogWriter())
app := os.Args[0]
args := os.Args[1:]
for _, arg := range args {
if arg == "-v" || arg == "--version" {
newArgs := make([]string, len(args)+1)
newArgs[0] = "version"
copy(newArgs[1:], args)
args = newArgs
break
}
if arg == "--generate-bash-completion" {
if len(args) > 1 {
// contextual auto complete
lastArg := args[len(args)-2]
switch lastArg {
case "-z": // zone
for _, zone := range ctx.SortedZones() {
fmt.Println(zone)
}
return
}
}
for name, _ := range commands {
fmt.Println(name)
}
return
}
}
c := cli.NewCLI(app, gafka.Version+"-"+gafka.BuildId)
c.Args = os.Args[1:]
if len(os.Args) > 1 {
// command given, convert alias
if alias, present := ctx.Alias(os.Args[1]); present {
var cargs []string
cargs = append(cargs, strings.Split(alias, " ")...)
if len(os.Args) > 2 {
cargs = append(cargs, os.Args[2:]...)
}
c.Args = cargs
}
}
c.Commands = commands
c.HelpFunc = func(m map[string]cli.CommandFactory) string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("A CLI tool for Zookeeper\n\n"))
buf.WriteString(cli.BasicHelpFunc(app)(m))
return buf.String()
}
exitCode, err := c.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "%+v\n", err)
os.Exit(1)
} else if c.IsVersion() {
os.Exit(0)
}
os.Exit(exitCode)
}