コード例 #1
0
ファイル: controller.go プロジェクト: funkygao/gafka
func New(zkzone *zk.ZkZone, listenAddr string, managerType string) Controller {
	// mysql cluster config
	b, err := zkzone.KatewayJobClusterConfig()
	if err != nil {
		panic(err)
	}
	var mcc = &config.ConfigMysql{}
	if err = mcc.From(b); err != nil {
		panic(err)
	}

	this := &controller{
		quiting:      make(chan struct{}),
		orchestrator: zkzone.NewOrchestrator(),
		mc:           mysql.New(mcc),
		ListenAddr:   listenAddr,
		Version:      gafka.BuildId,
	}
	this.ident, err = this.generateIdent()
	if err != nil {
		panic(err)
	}

	// hostname:95f333fb-731c-9c95-c598-8d6b99a9ec7d
	p := strings.SplitN(this.ident, ":", 2)
	this.shortId = fmt.Sprintf("%s:%s", p[0], this.ident[strings.LastIndexByte(this.ident, '-')+1:])
	this.setupAuditor()

	switch managerType {
	case "mysql":
		cf := mmysql.DefaultConfig(zkzone.Name())
		cf.Refresh = time.Minute * 5
		manager.Default = mmysql.New(cf)

	case "dummy":
		manager.Default = mdummy.New("")

	default:
		panic("unknown manager: " + managerType)
	}

	return this
}
コード例 #2
0
ファイル: actord.go プロジェクト: funkygao/gafka
func (this *WatchActord) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()

	b, err := this.Zkzone.KatewayJobClusterConfig()
	if err != nil {
		log.Error(err)
		return
	}
	var mcc = &config.ConfigMysql{}
	if err = mcc.From(b); err != nil {
		log.Error(err)
		return
	}

	this.mc = mysql.New(mcc)
	manager.Default = dummy.New("")
}
コード例 #3
0
ファイル: job.go プロジェクト: funkygao/gafka
func (this *Job) printResourcesAndActors() {
	lines := make([]string, 0)
	header := "Topic|Cluster|Ctime|Mtime"
	lines = append(lines, header)
	// jobs
	jobQueues := this.zkzone.ChildrenWithData(zk.PubsubJobQueues)
	sortedName := make([]string, 0, len(jobQueues))
	for name := range jobQueues {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	manager.Default = dummy.New("")
	for _, topic := range sortedName {
		zdata := jobQueues[topic]
		if appid := manager.Default.TopicAppid(topic); appid == "" {
			lines = append(lines, fmt.Sprintf("?%s|%s|%s|%s", topic,
				string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic,
				string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
		}

	}
	if len(lines) > 1 {
		this.Ui.Info("Jobs")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// webhooks
	lines = lines[:0]
	header = "Topic|Endpoints|Ctime|Mtime"
	lines = append(lines, header)
	webhoooks := this.zkzone.ChildrenWithData(zk.PubsubWebhooks)
	sortedName = sortedName[:0]
	for name := range webhoooks {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, topic := range sortedName {
		zdata := webhoooks[topic]
		var hook zk.WebhookMeta
		hook.From(zdata.Data())
		if appid := manager.Default.TopicAppid(topic); appid == "" {
			lines = append(lines, fmt.Sprintf("?%s|%+v|%s|%s", topic, hook.Endpoints,
				zdata.Ctime(), zdata.Mtime()))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%+v|%s|%s", topic, hook.Endpoints,
				zdata.Ctime(), zdata.Mtime()))
		}
	}
	if len(lines) > 1 {
		this.Ui.Info("Webhooks")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// actors
	lines = lines[:0]
	header = "Actor|Ctime|Mtime"
	lines = append(lines, header)
	actors := this.zkzone.ChildrenWithData(zk.PubsubActors)
	sortedName = sortedName[:0]
	for name := range actors {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, name := range sortedName {
		zdata := actors[name]
		lines = append(lines, fmt.Sprintf("%s|%s|%s", name,
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Actors")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// job <-> actor
	lines = lines[:0]
	header = "Topic|Actor|Ctime|Mtime"
	lines = append(lines, header)
	jobQueueOwners := this.zkzone.ChildrenWithData(zk.PubsubJobQueueOwners)
	sortedName = sortedName[:0]
	for name := range jobQueueOwners {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, jobQueue := range sortedName {
		zdata := jobQueueOwners[jobQueue]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", jobQueue, string(zdata.Data()),
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Job<->Actor")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// webhook <-> actor
	lines = lines[:0]
	header = "Topic|Actor|Ctime|Mtime"
	lines = append(lines, header)
	webhookOwners := this.zkzone.ChildrenWithData(zk.PubsubWebhookOwners)
	sortedName = sortedName[:0]
	for name := range webhookOwners {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, topic := range sortedName {
		zdata := webhookOwners[topic]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic, string(zdata.Data()),
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Webhook<->Actor")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}
コード例 #4
0
ファイル: gateway.go プロジェクト: funkygao/gafka
func New(id string) *Gateway {
	this := &Gateway{
		id:         id,
		shutdownCh: make(chan struct{}),
		quiting:    make(chan struct{}),
		certFile:   Options.CertFile,
		keyFile:    Options.KeyFile,
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(this.zkzone)
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
	rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
	if err != nil {
		log.Error("telemetry: %v", err)
	} else {
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
	}

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New(Options.DummyCluster)

	case "open":
		cf := manopen.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = manopen.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
		HttpHeaderAppid = "devid"
		HttpHeaderPubkey = "devsecret"
		HttpHeaderSubkey = "devsecret"

	default:
		panic("invalid manager store:" + Options.ManagerStore)
	}

	// initialize the servers on demand
	if Options.DebugHttpAddr != "" {
		this.debugMux = http.NewServeMux()
	}
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	} else {
		panic("manager server must be present")
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(Options.Debug)

		default:
			panic("invalid message store")
		}

		switch Options.JobStore {
		case "mysql":
			var mcc = &config.ConfigMysql{}
			b, err := this.zkzone.KatewayJobClusterConfig()
			if err != nil {
				panic(err)
			}
			if err = mcc.From(b); err != nil {
				panic(err)
			}
			jm, err := jobmysql.New(id, mcc)
			if err != nil {
				panic(fmt.Errorf("mysql job: %v", err))
			}

			job.Default = jm

		case "dummy":
			job.Default = jobdummy.New()

		default:
			panic("invalid job store")
		}

		// always create hh so that we can turn on/off it online
		switch Options.HintedHandoffType {
		case "disk":
			if len(Options.HintedHandoffDir) == 0 {
				panic("empty hh dir")
			}
			cfg := hhdisk.DefaultConfig()
			cfg.Dirs = strings.Split(Options.HintedHandoffDir, ",")
			if err := cfg.Validate(); err != nil {
				panic(err)
			}
			hhdisk.DisableBufio = !Options.HintedHandoffBufio
			if Options.AuditPub {
				hhdisk.Auditor = &this.pubServer.auditor
			}
			hh.Default = hhdisk.New(cfg)

		case "dummy":
			hh.Default = hhdummy.New()

		default:
			panic("unkown hinted handoff type")
		}

		if Options.FlushHintedOffOnly {
			meta.Default.Start()
			log.Trace("meta store[%s] started", meta.Default.Name())

			if err = store.DefaultPubStore.Start(); err != nil {
				panic(err)
			}
			log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

			hh.Default.FlushInflights()
			os.Exit(0)
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}
コード例 #5
0
ファイル: gateway.go プロジェクト: chendx79/gafka
func New(id string) *Gateway {
	this := &Gateway{
		id:           id,
		shutdownCh:   make(chan struct{}),
		certFile:     Options.CertFile,
		keyFile:      Options.KeyFile,
		clientStates: NewClientStates(),
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(Options.Zone, this.id, this.InstanceInfo())
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.guard = newGuard(this)
	this.timer = timewheel.NewTimeWheel(time.Second, 120)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New()

	default:
		panic("invalid manager store")
	}

	// initialize the servers on demand
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, &this.wg, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(&this.wg, Options.Debug)

		default:
			panic("invalid store")
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}