func (this *Mirror) Main() (exitCode int) { this.quit = make(chan struct{}) signal.RegisterHandler(func(sig os.Signal) { log.Info("received signal: %s", strings.ToUpper(sig.String())) log.Info("quiting...") this.once.Do(func() { close(this.quit) }) }, syscall.SIGINT, syscall.SIGTERM) limit := (1 << 20) * this.BandwidthLimit / 8 if this.BandwidthLimit > 0 { this.bandwidthRateLimiter = ratelimiter.NewLeakyBucket(limit*10, time.Second*10) } log.Info("starting mirror@%s", gafka.BuildId) // pprof debugAddr := ":10009" go http.ListenAndServe(debugAddr, nil) log.Info("pprof ready on %s", debugAddr) z1 := zk.NewZkZone(zk.DefaultConfig(this.Z1, ctx.ZoneZkAddrs(this.Z1))) z2 := zk.NewZkZone(zk.DefaultConfig(this.Z2, ctx.ZoneZkAddrs(this.Z2))) c1 := z1.NewCluster(this.C1) c2 := z2.NewCluster(this.C2) this.runMirror(c1, c2, limit) log.Info("bye mirror@%s, %s", gafka.BuildId, time.Since(this.startedAt)) log.Close() return }
func (this *Start) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("start", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.logfile, "log", defaultLogfile, "") cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&this.root, "p", defaultPrefix, "") cmdFlags.BoolVar(&this.debugMode, "d", false, "") cmdFlags.BoolVar(&this.forwardFor, "forwardfor", false, "") cmdFlags.IntVar(&this.pubPort, "pub", 10891, "") cmdFlags.IntVar(&this.subPort, "sub", 10892, "") cmdFlags.IntVar(&this.manPort, "man", 10893, "") cmdFlags.StringVar(&this.haproxyStatsUrl, "statsurl", "", "") cmdFlags.StringVar(&this.influxdbAddr, "influxaddr", "", "") cmdFlags.StringVar(&this.influxdbDbName, "influxdb", "", "") cmdFlags.StringVar(&this.httpAddr, "addr", ":10894", "monitor http server addr") if err := cmdFlags.Parse(args); err != nil { return 1 } lockFilename := fmt.Sprintf("%s/.lock", this.root) if locking.InstanceLocked(lockFilename) { panic(fmt.Sprintf("locked[%s] by another instance", lockFilename)) } locking.LockInstance(lockFilename) err := os.Chdir(this.root) swalllow(err) this.command = fmt.Sprintf("%s/sbin/haproxy", this.root) if _, err := os.Stat(this.command); err != nil { panic(err) } this.setupLogging(this.logfile, "info", "panic") this.starting = true this.startedAt = time.Now() if this.haproxyStatsUrl != "" && this.influxdbAddr != "" && this.influxdbDbName != "" { rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute) if err != nil { panic(err) } telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc) go func() { log.Info("telemetry started: %s", telemetry.Default.Name()) if err := telemetry.Default.Start(); err != nil { log.Error("telemetry: %v", err) } }() m := haproxyMetrics{ ctx: this, interval: time.Second * 30, uri: this.haproxyStatsUrl, } go m.start() } this.quitCh = make(chan struct{}) this.closed = make(chan struct{}) signal.RegisterHandler(func(sig os.Signal) { log.Info("ehaproxy[%s] got signal: %s", gafka.BuildId, strings.ToUpper(sig.String())) this.shutdown() log.Info("removing %s", configFile) os.Remove(configFile) log.Info("removing lock[%s]", lockFilename) locking.UnlockInstance(lockFilename) close(this.quitCh) if telemetry.Default != nil { log.Info("stopping telemetry and flush all metrics...") telemetry.Default.Stop() } log.Info("ehaproxy[%s] shutdown complete", gafka.BuildId) log.Info("ehaproxy[%s] %s bye!", gafka.BuildId, time.Since(this.startedAt)) close(this.closed) }, syscall.SIGINT, syscall.SIGTERM) this.main() <-this.closed log.Close() return }
// Main is the bootstrap main entry point, which will run for ever. func Main() { defer func() { if err := recover(); err != nil { fmt.Println(err) debug.PrintStack() } }() var err error zkzone := zk.NewZkZone(zk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone))) // meta pkg is required for store pkg metaConf := zkmeta.DefaultConfig() metaConf.Refresh = time.Minute * 5 meta.Default = zkmeta.New(metaConf, zkzone) meta.Default.Start() log.Trace("meta store[%s] started", meta.Default.Name()) if Options.InfluxAddr != "" && Options.InfluxDbname != "" { rc, err := influxdb.NewConfig(Options.InfluxAddr, Options.InfluxDbname, "", "", time.Minute) if err != nil { panic(err) } telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc) go func() { log.Info("telemetry[%s] started", telemetry.Default.Name()) if err := telemetry.Default.Start(); err != nil { log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err) } }() } else { log.Warn("empty influx flag, telemetry disabled") } store.DefaultPubStore = kafka.NewPubStore(100, 0, false, false, false) if err = store.DefaultPubStore.Start(); err != nil { panic(err) } log.Trace("pub store[%s] started", store.DefaultPubStore.Name()) c := controller.New(zkzone, Options.ListenAddr, Options.ManagerType) cfg := disk.DefaultConfig() cfg.Dirs = strings.Split(Options.HintedHandoffDir, ",") if err := cfg.Validate(); err != nil { panic(err) } disk.Auditor = c.Auditor() hh.Default = disk.New(cfg) if err = hh.Default.Start(); err != nil { panic(err) } log.Trace("hh[%s] started", hh.Default.Name()) signal.RegisterHandler(func(sig os.Signal) { log.Info("actord[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String())) log.Trace("controller[%s] stopping...", c.Id()) c.Stop() }, syscall.SIGINT, syscall.SIGTERM) if err = c.RunForever(); err != nil { panic(err) } log.Trace("pub store[%s] stopping", store.DefaultPubStore.Name()) store.DefaultPubStore.Stop() meta.Default.Stop() log.Trace("meta store[%s] stopped", meta.Default.Name()) if telemetry.Default != nil { telemetry.Default.Stop() log.Info("telemetry[%s] stopped", telemetry.Default.Name()) } zkzone.Close() log.Trace("zkzone stopped") log.Trace("all cleanup done") }
func (this *Gateway) Start() (err error) { log.Info("starting gateway[%s@%s]...", gafka.BuildId, gafka.BuiltAt) signal.RegisterHandler(func(sig os.Signal) { this.shutdownOnce.Do(func() { log.Info("gateway[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String())) close(this.quiting) }) }, syscall.SIGINT, syscall.SIGTERM) // yes we ignore HUP // keep watch on zk connection jitter go this.healthCheck() meta.Default.Start() log.Trace("meta store[%s] started", meta.Default.Name()) if err = manager.Default.Start(); err != nil { return } log.Trace("manager store[%s] started", manager.Default.Name()) if telemetry.Default != nil { go func() { log.Trace("telemetry[%s] started", telemetry.Default.Name()) if err = telemetry.Default.Start(); err != nil { log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err) } }() } if Options.EnableAccessLog { if err = this.accessLogger.Start(); err != nil { log.Error("access logger: %s", err) } } this.buildRouting() this.svrMetrics.Load() go startRuntimeMetrics(Options.ReporterInterval) // start up the servers this.manServer.Start() // man server is always present if this.pubServer != nil { if err = store.DefaultPubStore.Start(); err != nil { panic(err) } log.Trace("pub store[%s] started", store.DefaultPubStore.Name()) if err = hh.Default.Start(); err != nil { return } log.Trace("hh[%s] started", hh.Default.Name()) if err = job.Default.Start(); err != nil { panic(err) } log.Trace("job store[%s] started", job.Default.Name()) this.pubServer.Start() } if this.subServer != nil { if err = store.DefaultSubStore.Start(); err != nil { panic(err) } log.Trace("sub store[%s] started", store.DefaultSubStore.Name()) this.subServer.Start() } // the last thing is to register: notify others: come on baby! if registry.Default != nil { registry.Default.Register(this.id, this.InstanceInfo()) log.Info("gateway[%s:%s] ready, registered in %s :-)", ctx.Hostname(), this.id, registry.Default.Name()) } else { log.Info("gateway[%s:%s] ready, unregistered", ctx.Hostname(), this.id) } return nil }
func (this *Peek) Run(args []string) (exitCode int) { var ( cluster string zone string topicPattern string partitionId int wait time.Duration silence bool ) cmdFlags := flag.NewFlagSet("peek", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&cluster, "c", "", "") cmdFlags.StringVar(&topicPattern, "t", "", "") cmdFlags.IntVar(&partitionId, "p", 0, "") cmdFlags.BoolVar(&this.colorize, "color", true, "") cmdFlags.Int64Var(&this.lastN, "last", -1, "") cmdFlags.BoolVar(&this.pretty, "pretty", false, "") cmdFlags.IntVar(&this.limit, "n", -1, "") cmdFlags.StringVar(&this.column, "col", "", "") // TODO support multiple columns cmdFlags.BoolVar(&this.beep, "beep", false, "") cmdFlags.Int64Var(&this.offset, "offset", sarama.OffsetNewest, "") cmdFlags.BoolVar(&silence, "s", false, "") cmdFlags.DurationVar(&wait, "d", time.Hour, "") cmdFlags.BoolVar(&this.bodyOnly, "body", false, "") if err := cmdFlags.Parse(args); err != nil { return 1 } if this.pretty { this.bodyOnly = true } this.quit = make(chan struct{}) if silence { stats := newPeekStats() go stats.start() } zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) msgChan := make(chan *sarama.ConsumerMessage, 20000) // msg aggerator channel if cluster == "" { zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan) }) } else { zkcluster := zkzone.NewCluster(cluster) this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan) } signal.RegisterHandler(func(sig os.Signal) { log.Printf("received signal: %s", strings.ToUpper(sig.String())) log.Println("quiting...") this.once.Do(func() { close(this.quit) }) }, syscall.SIGINT, syscall.SIGTERM) var ( startAt = time.Now() msg *sarama.ConsumerMessage total int bytesN int64 ) var ( j map[string]interface{} prettyJSON bytes.Buffer ) LOOP: for { if time.Since(startAt) >= wait { this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s", gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt))) elapsed := time.Since(startAt).Seconds() if elapsed > 1. { this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed))) if total > 0 { this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total)))) } } return } select { case <-this.quit: this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s", gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt))) elapsed := time.Since(startAt).Seconds() if elapsed > 1. { this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed))) if total > 0 { this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total)))) } } return case <-time.After(time.Second): continue case msg = <-msgChan: if silence { stats.MsgCountPerSecond.Mark(1) stats.MsgBytesPerSecond.Mark(int64(len(msg.Value))) } else { var outmsg string if this.column != "" { if err := json.Unmarshal(msg.Value, &j); err != nil { this.Ui.Error(err.Error()) } else { var colVal string switch t := j[this.column].(type) { case string: colVal = t case float64: colVal = fmt.Sprintf("%.0f", t) case int: colVal = fmt.Sprintf("%d", t) } if this.bodyOnly { if this.pretty { if err = json.Indent(&prettyJSON, []byte(colVal), "", " "); err != nil { fmt.Println(err.Error()) } else { outmsg = string(prettyJSON.Bytes()) } } else { outmsg = colVal } } else if this.colorize { outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s", color.Green(msg.Topic), msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), colVal) } else { // colored UI will have invisible chars output outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s", msg.Topic, msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), colVal) } } } else { if this.bodyOnly { if this.pretty { json.Indent(&prettyJSON, msg.Value, "", " ") outmsg = string(prettyJSON.Bytes()) } else { outmsg = string(msg.Value) } } else if this.colorize { outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s", color.Green(msg.Topic), msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value)) } else { // colored UI will have invisible chars output outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s", msg.Topic, msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value)) } } if outmsg != "" { if this.beep { outmsg += "\a" } this.Ui.Output(outmsg) } } total++ bytesN += int64(len(msg.Value)) if this.limit > 0 && total >= this.limit { break LOOP } if this.lastN > 0 && total >= int(this.lastN) { break LOOP } } } return }
func (this *Mount) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("mount", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&this.cluster, "c", "", "") cmdFlags.StringVar(&this.logLevel, "l", "info", "") if err := cmdFlags.Parse(args); err != nil { return 1 } if validateArgs(this, this.Ui). on("-z", "-c"). invalid(args) { return 2 } this.mountPoint = args[len(args)-1] if !strings.HasPrefix(this.mountPoint, "/") { this.Ui.Error("mount point must start with /") return 1 } setupLogging("stdout", this.logLevel, "") c, err := fuse.Mount( this.mountPoint, fuse.FSName("kfs"), fuse.Subtype("kfs"), fuse.VolumeName("Kafka FS"), fuse.ReadOnly(), fuse.AllowOther(), ) if err != nil { log.Critical(err) } signal.RegisterHandler(func(sig os.Signal) { var err error for i := 0; i < 5; i++ { err = fuse.Unmount(this.mountPoint) if err == nil { break } log.Warn(err) time.Sleep(time.Second * 5) } if err == nil { log.Info("Kafka FS unmounted") } else { log.Error("Kafka FS unable to umount") } c.Close() os.Exit(0) }, syscall.SIGINT, syscall.SIGTERM) srv := fs.New(c, &fs.Config{}) fs := kfs.New(this.zone, this.cluster) if err := srv.Serve(fs); err != nil { log.Error(err) } <-c.Ready if err := c.MountError; err != nil { log.Error(err) } return }
func (this *Monitor) ServeForever() { defer this.zkzone.Close() this.startedAt = time.Now() log.Info("kguard[%s@%s] starting...", gafka.BuildId, gafka.BuiltAt) signal.RegisterHandler(func(sig os.Signal) { log.Info("kguard[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String())) this.Stop() close(this.quit) }, syscall.SIGINT, syscall.SIGTERM) // start the api server apiServer := &http.Server{ Addr: this.apiAddr, Handler: this.router, } apiListener, err := net.Listen("tcp4", this.apiAddr) if err == nil { log.Info("api http server ready on %s", this.apiAddr) go apiServer.Serve(apiListener) } else { panic(fmt.Sprintf("api http server: %v", err)) } backend, err := zookeeper.New(this.zkzone.ZkAddrList(), &store.Config{}) if err != nil { panic(err) } ip, _ := ctx.LocalIP() this.candidate = leadership.NewCandidate(backend, zk.KguardLeaderPath, ip.String(), 25*time.Second) electedCh, errCh := this.candidate.RunForElection() if err != nil { panic("Cannot run for election, store is probably down") } for { select { case isElected := <-electedCh: if isElected { log.Info("Won the election, starting all watchers") this.Start() } else { log.Warn("Fails the election, watching election events...") this.Stop() } case err := <-errCh: if err != nil { log.Error("Error during election: %v", err) } case <-this.quit: apiListener.Close() log.Info("api http server closed") log.Info("kguard[%s@%s] bye!", gafka.BuildId, gafka.BuiltAt) log.Close() return } } }