func New(zkzone *zk.ZkZone, listenAddr string, managerType string) Controller {
// mysql cluster config
b, err := zkzone.KatewayJobClusterConfig()
if err != nil {
panic(err)
}
var mcc = &config.ConfigMysql{}
if err = mcc.From(b); err != nil {
panic(err)
}
this := &controller{
quiting: make(chan struct{}),
orchestrator: zkzone.NewOrchestrator(),
mc: mysql.New(mcc),
ListenAddr: listenAddr,
Version: gafka.BuildId,
}
this.ident, err = this.generateIdent()
if err != nil {
panic(err)
}
// hostname:95f333fb-731c-9c95-c598-8d6b99a9ec7d
p := strings.SplitN(this.ident, ":", 2)
this.shortId = fmt.Sprintf("%s:%s", p[0], this.ident[strings.LastIndexByte(this.ident, '-')+1:])
this.setupAuditor()
switch managerType {
case "mysql":
cf := mmysql.DefaultConfig(zkzone.Name())
cf.Refresh = time.Minute * 5
manager.Default = mmysql.New(cf)
case "dummy":
manager.Default = mdummy.New("")
default:
panic("unknown manager: " + managerType)
}
return this
}
func (this *WatchActord) Init(ctx monitor.Context) {
this.Zkzone = ctx.ZkZone()
this.Stop = ctx.StopChan()
this.Wg = ctx.Inflight()
b, err := this.Zkzone.KatewayJobClusterConfig()
if err != nil {
log.Error(err)
return
}
var mcc = &config.ConfigMysql{}
if err = mcc.From(b); err != nil {
log.Error(err)
return
}
this.mc = mysql.New(mcc)
manager.Default = dummy.New("")
}
func (this *Job) printResourcesAndActors() {
lines := make([]string, 0)
header := "Topic|Cluster|Ctime|Mtime"
lines = append(lines, header)
// jobs
jobQueues := this.zkzone.ChildrenWithData(zk.PubsubJobQueues)
sortedName := make([]string, 0, len(jobQueues))
for name := range jobQueues {
sortedName = append(sortedName, name)
}
sort.Strings(sortedName)
manager.Default = dummy.New("")
for _, topic := range sortedName {
zdata := jobQueues[topic]
if appid := manager.Default.TopicAppid(topic); appid == "" {
lines = append(lines, fmt.Sprintf("?%s|%s|%s|%s", topic,
string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
} else {
lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic,
string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
}
}
if len(lines) > 1 {
this.Ui.Info("Jobs")
this.Ui.Output(columnize.SimpleFormat(lines))
}
// webhooks
lines = lines[:0]
header = "Topic|Endpoints|Ctime|Mtime"
lines = append(lines, header)
webhoooks := this.zkzone.ChildrenWithData(zk.PubsubWebhooks)
sortedName = sortedName[:0]
for name := range webhoooks {
sortedName = append(sortedName, name)
}
sort.Strings(sortedName)
for _, topic := range sortedName {
zdata := webhoooks[topic]
var hook zk.WebhookMeta
hook.From(zdata.Data())
if appid := manager.Default.TopicAppid(topic); appid == "" {
lines = append(lines, fmt.Sprintf("?%s|%+v|%s|%s", topic, hook.Endpoints,
zdata.Ctime(), zdata.Mtime()))
} else {
lines = append(lines, fmt.Sprintf("%s|%+v|%s|%s", topic, hook.Endpoints,
zdata.Ctime(), zdata.Mtime()))
}
}
if len(lines) > 1 {
this.Ui.Info("Webhooks")
this.Ui.Output(columnize.SimpleFormat(lines))
}
// actors
lines = lines[:0]
header = "Actor|Ctime|Mtime"
lines = append(lines, header)
actors := this.zkzone.ChildrenWithData(zk.PubsubActors)
sortedName = sortedName[:0]
for name := range actors {
sortedName = append(sortedName, name)
}
sort.Strings(sortedName)
for _, name := range sortedName {
zdata := actors[name]
lines = append(lines, fmt.Sprintf("%s|%s|%s", name,
zdata.Ctime(), zdata.Mtime()))
}
if len(lines) > 1 {
this.Ui.Info("Actors")
this.Ui.Output(columnize.SimpleFormat(lines))
}
// job <-> actor
lines = lines[:0]
header = "Topic|Actor|Ctime|Mtime"
lines = append(lines, header)
jobQueueOwners := this.zkzone.ChildrenWithData(zk.PubsubJobQueueOwners)
sortedName = sortedName[:0]
for name := range jobQueueOwners {
sortedName = append(sortedName, name)
}
sort.Strings(sortedName)
for _, jobQueue := range sortedName {
zdata := jobQueueOwners[jobQueue]
lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", jobQueue, string(zdata.Data()),
zdata.Ctime(), zdata.Mtime()))
}
if len(lines) > 1 {
this.Ui.Info("Job<->Actor")
this.Ui.Output(columnize.SimpleFormat(lines))
}
// webhook <-> actor
lines = lines[:0]
header = "Topic|Actor|Ctime|Mtime"
lines = append(lines, header)
webhookOwners := this.zkzone.ChildrenWithData(zk.PubsubWebhookOwners)
sortedName = sortedName[:0]
for name := range webhookOwners {
sortedName = append(sortedName, name)
}
sort.Strings(sortedName)
for _, topic := range sortedName {
zdata := webhookOwners[topic]
lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic, string(zdata.Data()),
zdata.Ctime(), zdata.Mtime()))
}
if len(lines) > 1 {
this.Ui.Info("Webhook<->Actor")
this.Ui.Output(columnize.SimpleFormat(lines))
}
}
func New(id string) *Gateway {
this := &Gateway{
id: id,
shutdownCh: make(chan struct{}),
quiting: make(chan struct{}),
certFile: Options.CertFile,
keyFile: Options.KeyFile,
}
this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
if err := this.zkzone.Ping(); err != nil {
panic(err)
}
if Options.EnableRegistry {
registry.Default = zk.New(this.zkzone)
}
metaConf := zkmeta.DefaultConfig()
metaConf.Refresh = Options.MetaRefresh
meta.Default = zkmeta.New(metaConf, this.zkzone)
this.accessLogger = NewAccessLogger("access_log", 100)
this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
if err != nil {
log.Error("telemetry: %v", err)
} else {
telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
}
// initialize the manager store
switch Options.ManagerStore {
case "mysql":
cf := mandb.DefaultConfig(Options.Zone)
cf.Refresh = Options.ManagerRefresh
manager.Default = mandb.New(cf)
manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
case "dummy":
manager.Default = mandummy.New(Options.DummyCluster)
case "open":
cf := manopen.DefaultConfig(Options.Zone)
cf.Refresh = Options.ManagerRefresh
manager.Default = manopen.New(cf)
manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
HttpHeaderAppid = "devid"
HttpHeaderPubkey = "devsecret"
HttpHeaderSubkey = "devsecret"
default:
panic("invalid manager store:" + Options.ManagerStore)
}
// initialize the servers on demand
if Options.DebugHttpAddr != "" {
this.debugMux = http.NewServeMux()
}
if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
Options.MaxClients, this)
} else {
panic("manager server must be present")
}
if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
Options.MaxClients, this)
switch Options.Store {
case "kafka":
store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
Options.UseCompress, Options.Debug, Options.DryRun)
case "dummy":
store.DefaultPubStore = storedummy.NewPubStore(Options.Debug)
default:
panic("invalid message store")
}
switch Options.JobStore {
case "mysql":
var mcc = &config.ConfigMysql{}
b, err := this.zkzone.KatewayJobClusterConfig()
if err != nil {
panic(err)
}
if err = mcc.From(b); err != nil {
panic(err)
}
jm, err := jobmysql.New(id, mcc)
if err != nil {
panic(fmt.Errorf("mysql job: %v", err))
}
job.Default = jm
case "dummy":
job.Default = jobdummy.New()
default:
panic("invalid job store")
}
// always create hh so that we can turn on/off it online
switch Options.HintedHandoffType {
case "disk":
if len(Options.HintedHandoffDir) == 0 {
panic("empty hh dir")
}
cfg := hhdisk.DefaultConfig()
cfg.Dirs = strings.Split(Options.HintedHandoffDir, ",")
if err := cfg.Validate(); err != nil {
panic(err)
}
hhdisk.DisableBufio = !Options.HintedHandoffBufio
if Options.AuditPub {
hhdisk.Auditor = &this.pubServer.auditor
}
hh.Default = hhdisk.New(cfg)
case "dummy":
hh.Default = hhdummy.New()
default:
panic("unkown hinted handoff type")
}
if Options.FlushHintedOffOnly {
meta.Default.Start()
log.Trace("meta store[%s] started", meta.Default.Name())
if err = store.DefaultPubStore.Start(); err != nil {
panic(err)
}
log.Trace("pub store[%s] started", store.DefaultPubStore.Name())
hh.Default.FlushInflights()
os.Exit(0)
}
}
if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
Options.MaxClients, this)
switch Options.Store {
case "kafka":
store.DefaultSubStore = storekfk.NewSubStore(this.subServer.closedConnCh, Options.Debug)
case "dummy":
store.DefaultSubStore = storedummy.NewSubStore(this.subServer.closedConnCh, Options.Debug)
default:
panic("invalid store")
}
}
return this
}
func New(id string) *Gateway {
this := &Gateway{
id: id,
shutdownCh: make(chan struct{}),
certFile: Options.CertFile,
keyFile: Options.KeyFile,
clientStates: NewClientStates(),
}
this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
if err := this.zkzone.Ping(); err != nil {
panic(err)
}
if Options.EnableRegistry {
registry.Default = zk.New(Options.Zone, this.id, this.InstanceInfo())
}
metaConf := zkmeta.DefaultConfig()
metaConf.Refresh = Options.MetaRefresh
meta.Default = zkmeta.New(metaConf, this.zkzone)
this.guard = newGuard(this)
this.timer = timewheel.NewTimeWheel(time.Second, 120)
this.accessLogger = NewAccessLogger("access_log", 100)
this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
// initialize the manager store
switch Options.ManagerStore {
case "mysql":
cf := mandb.DefaultConfig(Options.Zone)
cf.Refresh = Options.ManagerRefresh
manager.Default = mandb.New(cf)
manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
case "dummy":
manager.Default = mandummy.New()
default:
panic("invalid manager store")
}
// initialize the servers on demand
if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
Options.MaxClients, this)
}
if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
Options.MaxClients, this)
switch Options.Store {
case "kafka":
store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
Options.UseCompress, &this.wg, Options.Debug, Options.DryRun)
case "dummy":
store.DefaultPubStore = storedummy.NewPubStore(&this.wg, Options.Debug)
default:
panic("invalid store")
}
}
if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
Options.MaxClients, this)
switch Options.Store {
case "kafka":
store.DefaultSubStore = storekfk.NewSubStore(&this.wg,
this.subServer.closedConnCh, Options.Debug)
case "dummy":
store.DefaultSubStore = storedummy.NewSubStore(&this.wg,
this.subServer.closedConnCh, Options.Debug)
default:
panic("invalid store")
}
}
return this
}