コード例 #1
0
ファイル: monitor.go プロジェクト: funkygao/gafka
func (this *Monitor) Init() {
	var logFile, zone string
	flag.StringVar(&logFile, "log", "stdout", "log filename")
	flag.StringVar(&zone, "z", "", "zone, required")
	flag.StringVar(&this.apiAddr, "http", ":10025", "api http server addr")
	flag.StringVar(&this.influxdbAddr, "influxAddr", "", "influxdb addr, required")
	flag.StringVar(&this.influxdbDbName, "db", "", "influxdb db name, required")
	flag.StringVar(&this.externalDir, "confd", "", "external script config dir")
	flag.Parse()

	if zone == "" || this.influxdbDbName == "" || this.influxdbAddr == "" {
		panic("zone or influxdb empty, run help ")
	}

	ctx.LoadFromHome()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.watchers = make([]Watcher, 0, 10)
	this.quit = make(chan struct{})

	// export RESTful api
	this.setupRoutes()

	if logFile == "stdout" {
		log.AddFilter("stdout", log.TRACE, log.NewConsoleLogWriter())
	} else {
		log.DeleteFilter("stdout")

		filer := log.NewFileLogWriter(logFile, true, false, 0644)
		filer.SetRotateDaily(true)
		filer.SetFormat("[%d %T] [%L] (%S) %M")
		log.AddFilter("file", log.TRACE, filer)
	}

	rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute)
	if err != nil {
		panic(err)
	}
	telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
}
コード例 #2
0
ファイル: start.go プロジェクト: funkygao/gafka
func (this *Start) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("start", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.logfile, "log", defaultLogfile, "")
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.root, "p", defaultPrefix, "")
	cmdFlags.BoolVar(&this.debugMode, "d", false, "")
	cmdFlags.BoolVar(&this.forwardFor, "forwardfor", false, "")
	cmdFlags.IntVar(&this.pubPort, "pub", 10891, "")
	cmdFlags.IntVar(&this.subPort, "sub", 10892, "")
	cmdFlags.IntVar(&this.manPort, "man", 10893, "")
	cmdFlags.StringVar(&this.haproxyStatsUrl, "statsurl", "", "")
	cmdFlags.StringVar(&this.influxdbAddr, "influxaddr", "", "")
	cmdFlags.StringVar(&this.influxdbDbName, "influxdb", "", "")
	cmdFlags.StringVar(&this.httpAddr, "addr", ":10894", "monitor http server addr")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	lockFilename := fmt.Sprintf("%s/.lock", this.root)
	if locking.InstanceLocked(lockFilename) {
		panic(fmt.Sprintf("locked[%s] by another instance", lockFilename))
	}

	locking.LockInstance(lockFilename)

	err := os.Chdir(this.root)
	swalllow(err)

	this.command = fmt.Sprintf("%s/sbin/haproxy", this.root)
	if _, err := os.Stat(this.command); err != nil {
		panic(err)
	}

	this.setupLogging(this.logfile, "info", "panic")
	this.starting = true
	this.startedAt = time.Now()

	if this.haproxyStatsUrl != "" &&
		this.influxdbAddr != "" && this.influxdbDbName != "" {
		rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute)
		if err != nil {
			panic(err)
		}
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
		go func() {
			log.Info("telemetry started: %s", telemetry.Default.Name())

			if err := telemetry.Default.Start(); err != nil {
				log.Error("telemetry: %v", err)
			}
		}()

		m := haproxyMetrics{
			ctx:      this,
			interval: time.Second * 30,
			uri:      this.haproxyStatsUrl,
		}
		go m.start()
	}

	this.quitCh = make(chan struct{})
	this.closed = make(chan struct{})
	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("ehaproxy[%s] got signal: %s", gafka.BuildId, strings.ToUpper(sig.String()))
		this.shutdown()

		log.Info("removing %s", configFile)
		os.Remove(configFile)

		log.Info("removing lock[%s]", lockFilename)
		locking.UnlockInstance(lockFilename)

		close(this.quitCh)

		if telemetry.Default != nil {
			log.Info("stopping telemetry and flush all metrics...")
			telemetry.Default.Stop()
		}

		log.Info("ehaproxy[%s] shutdown complete", gafka.BuildId)
		log.Info("ehaproxy[%s] %s bye!", gafka.BuildId, time.Since(this.startedAt))

		close(this.closed)
	}, syscall.SIGINT, syscall.SIGTERM)

	this.main()

	<-this.closed
	log.Close()

	return
}
コード例 #3
0
ファイル: bootstrap.go プロジェクト: funkygao/gafka
// Main is the bootstrap main entry point, which will run for ever.
func Main() {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
			debug.PrintStack()
		}
	}()

	var err error
	zkzone := zk.NewZkZone(zk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))

	// meta pkg is required for store pkg
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = time.Minute * 5
	meta.Default = zkmeta.New(metaConf, zkzone)
	meta.Default.Start()
	log.Trace("meta store[%s] started", meta.Default.Name())

	if Options.InfluxAddr != "" && Options.InfluxDbname != "" {
		rc, err := influxdb.NewConfig(Options.InfluxAddr, Options.InfluxDbname, "", "", time.Minute)
		if err != nil {
			panic(err)
		}
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
		go func() {
			log.Info("telemetry[%s] started", telemetry.Default.Name())

			if err := telemetry.Default.Start(); err != nil {
				log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err)
			}
		}()
	} else {
		log.Warn("empty influx flag, telemetry disabled")
	}

	store.DefaultPubStore = kafka.NewPubStore(100, 0, false, false, false)
	if err = store.DefaultPubStore.Start(); err != nil {
		panic(err)
	}
	log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

	c := controller.New(zkzone, Options.ListenAddr, Options.ManagerType)

	cfg := disk.DefaultConfig()
	cfg.Dirs = strings.Split(Options.HintedHandoffDir, ",")
	if err := cfg.Validate(); err != nil {
		panic(err)
	}
	disk.Auditor = c.Auditor()
	hh.Default = disk.New(cfg)
	if err = hh.Default.Start(); err != nil {
		panic(err)
	}
	log.Trace("hh[%s] started", hh.Default.Name())

	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("actord[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String()))

		log.Trace("controller[%s] stopping...", c.Id())
		c.Stop()

	}, syscall.SIGINT, syscall.SIGTERM)

	if err = c.RunForever(); err != nil {
		panic(err)
	}

	log.Trace("pub store[%s] stopping", store.DefaultPubStore.Name())
	store.DefaultPubStore.Stop()

	meta.Default.Stop()
	log.Trace("meta store[%s] stopped", meta.Default.Name())

	if telemetry.Default != nil {
		telemetry.Default.Stop()
		log.Info("telemetry[%s] stopped", telemetry.Default.Name())
	}

	zkzone.Close()
	log.Trace("zkzone stopped")

	log.Trace("all cleanup done")
}
コード例 #4
0
ファイル: gateway.go プロジェクト: funkygao/gafka
func New(id string) *Gateway {
	this := &Gateway{
		id:         id,
		shutdownCh: make(chan struct{}),
		quiting:    make(chan struct{}),
		certFile:   Options.CertFile,
		keyFile:    Options.KeyFile,
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(this.zkzone)
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
	rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
	if err != nil {
		log.Error("telemetry: %v", err)
	} else {
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
	}

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New(Options.DummyCluster)

	case "open":
		cf := manopen.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = manopen.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
		HttpHeaderAppid = "devid"
		HttpHeaderPubkey = "devsecret"
		HttpHeaderSubkey = "devsecret"

	default:
		panic("invalid manager store:" + Options.ManagerStore)
	}

	// initialize the servers on demand
	if Options.DebugHttpAddr != "" {
		this.debugMux = http.NewServeMux()
	}
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	} else {
		panic("manager server must be present")
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(Options.Debug)

		default:
			panic("invalid message store")
		}

		switch Options.JobStore {
		case "mysql":
			var mcc = &config.ConfigMysql{}
			b, err := this.zkzone.KatewayJobClusterConfig()
			if err != nil {
				panic(err)
			}
			if err = mcc.From(b); err != nil {
				panic(err)
			}
			jm, err := jobmysql.New(id, mcc)
			if err != nil {
				panic(fmt.Errorf("mysql job: %v", err))
			}

			job.Default = jm

		case "dummy":
			job.Default = jobdummy.New()

		default:
			panic("invalid job store")
		}

		// always create hh so that we can turn on/off it online
		switch Options.HintedHandoffType {
		case "disk":
			if len(Options.HintedHandoffDir) == 0 {
				panic("empty hh dir")
			}
			cfg := hhdisk.DefaultConfig()
			cfg.Dirs = strings.Split(Options.HintedHandoffDir, ",")
			if err := cfg.Validate(); err != nil {
				panic(err)
			}
			hhdisk.DisableBufio = !Options.HintedHandoffBufio
			if Options.AuditPub {
				hhdisk.Auditor = &this.pubServer.auditor
			}
			hh.Default = hhdisk.New(cfg)

		case "dummy":
			hh.Default = hhdummy.New()

		default:
			panic("unkown hinted handoff type")
		}

		if Options.FlushHintedOffOnly {
			meta.Default.Start()
			log.Trace("meta store[%s] started", meta.Default.Name())

			if err = store.DefaultPubStore.Start(); err != nil {
				panic(err)
			}
			log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

			hh.Default.FlushInflights()
			os.Exit(0)
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}