Example #1
0
func (this *Gateway) InstanceInfo() []byte {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}
	info := gzk.KatewayMeta{
		Id:        this.id,
		Zone:      Options.Zone,
		Ver:       gafka.Version,
		Build:     gafka.BuildId,
		BuiltAt:   gafka.BuiltAt,
		Host:      ctx.Hostname(),
		Ip:        ip.String(),
		Cpu:       ctx.NumCPUStr(),
		Arch:      fmt.Sprintf("%s:%s-%s/%s", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH),
		PubAddr:   Options.PubHttpAddr,
		SPubAddr:  Options.PubHttpsAddr,
		SubAddr:   Options.SubHttpAddr,
		SSubAddr:  Options.SubHttpsAddr,
		ManAddr:   Options.ManHttpAddr,
		SManAddr:  Options.ManHttpsAddr,
		DebugAddr: Options.DebugHttpAddr,
	}
	d, _ := json.Marshal(info)
	return d
}
Example #2
0
func init() {
	ip, _ := ctx.LocalIP()
	flag.StringVar(&addr, "addr", fmt.Sprintf("%s:9192", ip), "sub kateway addr")
	flag.StringVar(&group, "g", "group1", "consumer group name")
	flag.StringVar(&appid, "appid", "app1", "consume whose topic")
	flag.IntVar(&step, "step", 1, "display progress step")
	flag.StringVar(&topic, "t", "foobar", "topic to sub")
	flag.DurationVar(&sleep, "sleep", 0, "sleep between pub")
	flag.IntVar(&n, "n", 1000000, "run sub how many times")
	flag.Parse()
}
Example #3
0
func init() {
	ip, _ := ctx.LocalIP()
	flag.IntVar(&c, "c", 10, "client concurrency")
	flag.IntVar(&msgSize, "sz", 100, "msg size")
	flag.StringVar(&appid, "appid", "app1", "app id")
	flag.DurationVar(&sleep, "sleep", 0, "sleep between pub")
	flag.StringVar(&addr, "h", fmt.Sprintf("%s:9191", ip.String()), "pub http addr")
	flag.Int64Var(&step, "step", 1, "display progress step")
	flag.StringVar(&key, "key", "", "job key")
	flag.BoolVar(&debug, "debug", false, "debug")
	flag.StringVar(&tag, "tag", "", "add tag to each job")
	flag.StringVar(&topic, "t", "foobar", "topic to pub")
	flag.StringVar(&workerId, "id", "1", "worker id")
	flag.Parse()
}
Example #4
0
func init() {
	ip, _ := ctx.LocalIP()
	flag.StringVar(&addr, "addr", fmt.Sprintf("%s:9192", ip), "sub kateway addr")
	flag.StringVar(&group, "g", "group1", "consumer group name")
	flag.StringVar(&appid, "appid", "app1", "consume whose topic")
	flag.IntVar(&step, "step", 1, "display progress step")
	flag.StringVar(&mode, "mode", "subx", "sub mode")
	flag.StringVar(&topic, "t", "foobar", "topic to sub")
	flag.IntVar(&c, "c", 1, "concurrent to simulate race condition")
	flag.StringVar(&tag, "tag", "", "tag filter")
	flag.DurationVar(&sleep, "sleep", 0, "sleep between pub")
	flag.IntVar(&n, "n", 1000000, "run sub how many times")
	flag.Parse()

	rd.RandSeedWithTime()
}
Example #5
0
func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	flag.IntVar(&c, "c", 10, "client concurrency")
	flag.IntVar(&msgSize, "sz", 100, "msg size")
	flag.StringVar(&appid, "appid", "app1", "app id")
	flag.DurationVar(&sleep, "sleep", 0, "sleep between pub")
	ip, _ := ctx.LocalIP()
	flag.StringVar(&addr, "h", fmt.Sprintf("%s:9191", ip.String()), "pub http addr")
	flag.Int64Var(&step, "step", 1, "display progress step")
	flag.StringVar(&key, "key", "", "message key")
	flag.BoolVar(&debug, "debug", false, "debug")
	flag.BoolVar(&tag, "tag", false, "add random tag to each message")
	flag.StringVar(&topic, "t", "foobar", "topic to pub")
	flag.StringVar(&workerId, "id", "1", "worker id")
	flag.Parse()

	for i := 0; i < c; i++ {
		go pubGatewayLoop(i)
	}

	select {}
}
Example #6
0
func (this *Deploy) demo() {
	var (
		maxPort int

		myPort     = -1
		myBrokerId = -1
	)

	this.zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		maxBrokerId := -1
		for _, broker := range liveBrokers {
			if maxPort < broker.Port {
				maxPort = broker.Port
			}

			if cluster == this.cluster {
				myPort = broker.Port

				bid, _ := strconv.Atoi(broker.Id)
				if bid > maxBrokerId {
					maxBrokerId = bid
				}

				myBrokerId = maxBrokerId + 1 // next deployable broker id
			}
		}

	})

	ip, err := ctx.LocalIP()
	swallow(err)

	if myPort == -1 {
		// the 1st deployment of this cluster
		myPort = maxPort + 1
	}
	if myBrokerId == -1 {
		// 1st broker id starts with 0
		myBrokerId = 0
	}
	logDirs := make([]string, 0)
	for i := 0; i <= 15; i++ {
		logDir := fmt.Sprintf("/data%d/%s", i, this.clusterName())
		if gio.DirExists(filepath.Dir(logDir)) {
			logDirs = append(logDirs, logDir)
		}
	}
	if len(logDirs) == 0 {
		// deploy on a small disk host, having no /dataX dirs
		logDirs = []string{fmt.Sprintf("%s/logs", this.instanceDir())}
	}

	influxAddr := ctx.Zone(this.zone).InfluxAddr
	if influxAddr != "" {
		this.Ui.Output(fmt.Sprintf("gk deploy -z %s -c %s -broker.id %d -port %d -ip %s -log.dirs %s -influx %s",
			this.zone, this.cluster,
			myBrokerId, myPort,
			ip.String(),
			strings.Join(logDirs, ","),
			influxAddr))
	} else {
		this.Ui.Output(fmt.Sprintf("gk deploy -z %s -c %s -broker.id %d -port %d -ip %s -log.dirs %s",
			this.zone, this.cluster,
			myBrokerId, myPort,
			ip.String(),
			strings.Join(logDirs, ",")))
	}

}
Example #7
0
func ParseFlags() {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}

	var (
		defaultPubHttpAddr  = fmt.Sprintf("%s:9191", ip.String())
		defaultPubHttpsAddr = ""
		defaultSubHttpAddr  = fmt.Sprintf("%s:9192", ip.String())
		defaultSubHttpsAddr = ""
		defaultManHttpAddr  = fmt.Sprintf("%s:9193", ip.String())
		defaultManHttpsAddr = ""
	)

	flag.StringVar(&Options.Id, "id", "", "kateway id, the id must be unique within a host")
	flag.StringVar(&Options.Zone, "zone", "", "kafka zone name")
	flag.StringVar(&Options.PubHttpAddr, "pubhttp", defaultPubHttpAddr, "pub http bind addr")
	flag.StringVar(&Options.PubHttpsAddr, "pubhttps", defaultPubHttpsAddr, "pub https bind addr")
	flag.StringVar(&Options.SubHttpAddr, "subhttp", defaultSubHttpAddr, "sub http bind addr")
	flag.StringVar(&Options.SubHttpsAddr, "subhttps", defaultSubHttpsAddr, "sub https bind addr")
	flag.StringVar(&Options.ManHttpAddr, "manhttp", defaultManHttpAddr, "management http bind addr")
	flag.StringVar(&Options.ManHttpsAddr, "manhttps", defaultManHttpsAddr, "management https bind addr")
	flag.StringVar(&Options.LogLevel, "level", "trace", "log level")
	flag.StringVar(&Options.LogFile, "log", "stdout", "log file, default stdout")
	flag.StringVar(&Options.CrashLogFile, "crashlog", "", "crash log")
	flag.StringVar(&Options.CertFile, "certfile", "", "cert file path")
	flag.StringVar(&Options.PidFile, "pid", "", "pid file")
	flag.StringVar(&Options.KeyFile, "keyfile", "", "key file path")
	flag.StringVar(&Options.DebugHttpAddr, "debughttp", "", "debug http bind addr")
	flag.StringVar(&Options.Store, "store", "kafka", "backend store")
	flag.StringVar(&Options.ManagerStore, "mstore", "mysql", "store integration with manager")
	flag.StringVar(&Options.ConfigFile, "conf", "/etc/kateway.cf", "config file")
	flag.StringVar(&Options.KillFile, "kill", "", "kill running kateway by pid file")
	flag.StringVar(&Options.InfluxServer, "influxdbaddr", "http://10.77.144.193:10036", "influxdb server address for the metrics reporter")
	flag.StringVar(&Options.InfluxDbName, "influxdbname", "pubsub", "influxdb db name")
	flag.BoolVar(&Options.ShowVersion, "version", false, "show version and exit")
	flag.BoolVar(&Options.Debug, "debug", false, "enable debug mode")
	flag.BoolVar(&Options.GolangTrace, "gotrace", false, "go tool trace")
	flag.BoolVar(&Options.EnableAccessLog, "accesslog", false, "en(dis)able access log")
	flag.BoolVar(&Options.EnableClientStats, "clientsmap", false, "record online pub/sub clients")
	flag.BoolVar(&Options.DryRun, "dryrun", false, "dry run mode")
	flag.BoolVar(&Options.PermitUnregisteredGroup, "unregrp", false, "permit sub group usage without being registered")
	flag.BoolVar(&Options.PermitStandbySub, "standbysub", false, "permits sub threads exceed partitions")
	flag.BoolVar(&Options.EnableGzip, "gzip", true, "enable http response gzip")
	flag.BoolVar(&Options.CpuAffinity, "cpuaffinity", false, "enable cpu affinity")
	flag.BoolVar(&Options.Ratelimit, "raltelimit", false, "enable rate limit")
	flag.BoolVar(&Options.EnableHttpPanicRecover, "httppanic", false, "enable http handler panic recover")
	flag.BoolVar(&Options.DisableMetrics, "metricsoff", false, "disable metrics reporter")
	flag.IntVar(&Options.HttpHeaderMaxBytes, "maxheader", 4<<10, "http header max size in bytes")
	flag.Int64Var(&Options.MaxPubSize, "maxpub", 512<<10, "max Pub message size")
	flag.IntVar(&Options.MinPubSize, "minpub", 1, "min Pub message size")
	flag.IntVar(&Options.MaxPubRetries, "pubretry", 5, "max retries when Pub fails")
	flag.IntVar(&Options.MaxRequestPerConn, "maxreq", -1, "max request per connection")
	flag.IntVar(&Options.MaxMsgTagLen, "tagsz", 120, "max message tag length permitted")
	flag.IntVar(&Options.LogRotateSize, "logsize", 10<<30, "max unrotated log file size")
	flag.IntVar(&Options.PubQpsLimit, "publimit", 60*10000, "pub qps limit per minute per ip")
	flag.IntVar(&Options.PubPoolCapcity, "pubpool", 100, "pub connection pool capacity")
	flag.IntVar(&Options.MaxClients, "maxclient", 100000, "max concurrent connections")
	flag.DurationVar(&Options.OffsetCommitInterval, "offsetcommit", time.Minute, "consumer offset commit interval")
	flag.DurationVar(&Options.HttpReadTimeout, "httprtimeout", time.Minute*5, "http server read timeout")
	flag.DurationVar(&Options.HttpWriteTimeout, "httpwtimeout", time.Minute, "http server write timeout")
	flag.DurationVar(&Options.SubTimeout, "subtimeout", time.Second*30, "sub timeout before send http 204")
	flag.DurationVar(&Options.ReporterInterval, "report", time.Second*10, "reporter flush interval")
	flag.DurationVar(&Options.MetaRefresh, "metarefresh", time.Minute*10, "meta data refresh interval")
	flag.DurationVar(&Options.ManagerRefresh, "manrefresh", time.Minute*5, "manager integration refresh interval")
	flag.DurationVar(&Options.ConsoleMetricsInterval, "consolemetrics", 0, "console metrics report interval")
	flag.DurationVar(&Options.PubPoolIdleTimeout, "pubpoolidle", 0, "pub pool connect idle timeout")

	flag.Parse()
}
Example #8
0
func (this *Monitor) ServeForever() {
	defer this.zkzone.Close()

	this.startedAt = time.Now()
	log.Info("kguard[%s@%s] starting...", gafka.BuildId, gafka.BuiltAt)

	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("kguard[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String()))

		this.Stop()
		close(this.quit)
	}, syscall.SIGINT, syscall.SIGTERM)

	// start the api server
	apiServer := &http.Server{
		Addr:    this.apiAddr,
		Handler: this.router,
	}
	apiListener, err := net.Listen("tcp4", this.apiAddr)
	if err == nil {
		log.Info("api http server ready on %s", this.apiAddr)
		go apiServer.Serve(apiListener)
	} else {
		panic(fmt.Sprintf("api http server: %v", err))
	}

	backend, err := zookeeper.New(this.zkzone.ZkAddrList(), &store.Config{})
	if err != nil {
		panic(err)
	}

	ip, _ := ctx.LocalIP()
	this.candidate = leadership.NewCandidate(backend, zk.KguardLeaderPath, ip.String(), 25*time.Second)
	electedCh, errCh := this.candidate.RunForElection()
	if err != nil {
		panic("Cannot run for election, store is probably down")
	}

	for {
		select {
		case isElected := <-electedCh:
			if isElected {
				log.Info("Won the election, starting all watchers")

				this.Start()
			} else {
				log.Warn("Fails the election, watching election events...")
				this.Stop()
			}

		case err := <-errCh:
			if err != nil {
				log.Error("Error during election: %v", err)
			}

		case <-this.quit:
			apiListener.Close()
			log.Info("api http server closed")
			log.Info("kguard[%s@%s] bye!", gafka.BuildId, gafka.BuiltAt)
			log.Close()
			return
		}
	}

}
Example #9
0
func ParseFlags() {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}

	var (
		defaultPubHttpAddr  = fmt.Sprintf("%s:9191", ip.String())
		defaultPubHttpsAddr = ""
		defaultSubHttpAddr  = fmt.Sprintf("%s:9192", ip.String())
		defaultSubHttpsAddr = ""
		defaultManHttpAddr  = fmt.Sprintf("%s:9193", ip.String())
		defaultManHttpsAddr = ""
	)

	flag.StringVar(&Options.Id, "id", "", "kateway id, the id must be unique within a host")
	flag.StringVar(&Options.Zone, "zone", "", "kafka zone name")
	flag.StringVar(&Options.PubHttpAddr, "pubhttp", defaultPubHttpAddr, "pub http bind addr")
	flag.StringVar(&Options.PubHttpsAddr, "pubhttps", defaultPubHttpsAddr, "pub https bind addr")
	flag.StringVar(&Options.SubHttpAddr, "subhttp", defaultSubHttpAddr, "sub http bind addr")
	flag.StringVar(&Options.SubHttpsAddr, "subhttps", defaultSubHttpsAddr, "sub https bind addr")
	flag.StringVar(&Options.ManHttpAddr, "manhttp", defaultManHttpAddr, "management http bind addr")
	flag.StringVar(&Options.ManHttpsAddr, "manhttps", defaultManHttpsAddr, "management https bind addr")
	flag.StringVar(&Options.LogLevel, "level", "trace", "log level")
	flag.StringVar(&Options.LogFile, "log", "stdout", "log file, default stdout")
	flag.StringVar(&Options.CrashLogFile, "crashlog", "", "crash log")
	flag.StringVar(&Options.CertFile, "certfile", "", "cert file path")
	flag.StringVar(&Options.PidFile, "pid", "", "pid file")
	flag.StringVar(&Options.KeyFile, "keyfile", "", "key file path")
	flag.StringVar(&Options.DebugHttpAddr, "debughttp", "", "debug http bind addr")
	flag.StringVar(&Options.Store, "store", "kafka", "message underlying store")
	flag.StringVar(&Options.HintedHandoffType, "hhtype", "disk", "underlying hinted handoff")
	flag.StringVar(&Options.HintedHandoffDir, "hhdirs", "hhdata", "hinted handoff dirs seperated by comma")
	flag.BoolVar(&Options.FlushHintedOffOnly, "hhflush", false, "flush hinted handoff and exit")
	flag.StringVar(&Options.JobStore, "jstore", "mysql", "job underlying store")
	flag.StringVar(&Options.DummyCluster, "dummycluster", "me", "dummy store's cluster name")
	flag.StringVar(&Options.ManagerStore, "mstore", "mysql", "store integration with manager")
	flag.StringVar(&Options.ConfigFile, "conf", "", "config file, defaults $HOME/.gafka.cf")
	flag.StringVar(&Options.KillFile, "kill", "", "kill running kateway by pid file")
	flag.StringVar(&Options.InfluxServer, "influxdbaddr", "", "influxdb server address for the metrics reporter")
	flag.StringVar(&Options.InfluxDbName, "influxdbname", "pubsub", "influxdb db name")
	flag.BoolVar(&Options.ShowVersion, "version", false, "show version and exit")
	flag.BoolVar(&Options.Debug, "debug", false, "enable debug mode")
	flag.BoolVar(&Options.RunSwaggerServer, "swagger", false, "run swagger server")
	flag.BoolVar(&Options.GolangTrace, "gotrace", false, "go tool trace")
	flag.BoolVar(&Options.AllwaysHintedHandoff, "allhh", false, "always use hh")
	flag.BoolVar(&Options.AuditPub, "auditpub", true, "enable Pub audit")
	flag.BoolVar(&Options.AuditSub, "auditsub", true, "enable Sub audit")
	flag.BoolVar(&Options.UseCompress, "snappy", false, "backend store will snappy compress messages")
	flag.BoolVar(&Options.EnableAccessLog, "accesslog", false, "en(dis)able access log")
	flag.BoolVar(&Options.EnableRegistry, "withreg", true, "self register in zk, otherwise isolated from cluster")
	flag.BoolVar(&Options.DryRun, "dryrun", false, "dry run mode")
	flag.BoolVar(&Options.HintedHandoffBufio, "hhbuf", false, "enable hinted handoff bufio")
	flag.BoolVar(&Options.EnableHintedHandoff, "hh", true, "enable hinted handoff for full pub availability")
	flag.BoolVar(&Options.PermitUnregisteredGroup, "unregrp", false, "permit sub group usage without being registered")
	flag.BoolVar(&Options.PermitStandbySub, "standbysub", false, "permits sub threads exceed partitions")
	flag.BoolVar(&Options.EnableGzip, "gzip", false, "enable http response gzip")
	flag.BoolVar(&Options.CpuAffinity, "cpuaffinity", false, "enable cpu affinity")
	flag.BoolVar(&Options.BadGroupRateLimit, "badgroup_rater", true, "rate limit of bad consumer group")
	flag.BoolVar(&Options.BadPubAppRateLimit, "badpub_rater", true, "rate limit of bad pub app client")
	flag.BoolVar(&Options.Ratelimit, "raltelimit", false, "enable rate limit")
	flag.BoolVar(&Options.EnableHttpPanicRecover, "httppanic", true, "enable http handler panic recover")
	flag.BoolVar(&Options.DisableMetrics, "metricsoff", false, "disable metrics reporter")
	flag.IntVar(&Options.HttpHeaderMaxBytes, "maxheader", 4<<10, "http header max size in bytes")
	flag.Int64Var(&Options.MaxPubSize, "maxpub", 512<<10, "max Pub message size")
	flag.Int64Var(&Options.MaxJobSize, "maxjob", 16<<10, "max Pub job size")
	flag.IntVar(&Options.MinPubSize, "minpub", 1, "min Pub message size")
	flag.IntVar(&Options.MaxRequestPerConn, "maxreq", -1, "max request per connection")
	flag.IntVar(&Options.AssignJobShardId, "shardid", 1, "how to assign shard id for new app")
	flag.IntVar(&Options.MaxMsgTagLen, "tagsz", 1024, "max message tag length permitted")
	// kafka Fetch maxFetchSize=1MB, so if our msg agv size is 250B, batch size can be 4000
	flag.IntVar(&Options.MaxSubBatchSize, "maxbatch", 4000, "max sub batch size")
	flag.IntVar(&Options.LogRotateSize, "logsize", 10<<30, "max unrotated log file size")
	flag.Int64Var(&Options.PubQpsLimit, "publimit", 60*10000, "pub qps limit per minute per ip")
	flag.IntVar(&Options.PubPoolCapcity, "pubpool", 100, "pub connection pool capacity")
	flag.IntVar(&Options.MaxClients, "maxclient", 100000, "max concurrent connections")
	flag.DurationVar(&Options.OffsetCommitInterval, "offsetcommit", time.Minute, "consumer offset commit interval")
	flag.DurationVar(&Options.HttpReadTimeout, "httprtimeout", time.Minute*5, "http server read timeout")
	flag.DurationVar(&Options.HttpWriteTimeout, "httpwtimeout", time.Minute, "http server write timeout")
	flag.DurationVar(&Options.SubTimeout, "subtimeout", time.Second*30, "sub timeout before send http 204")
	flag.DurationVar(&Options.ReporterInterval, "report", time.Second*30, "reporter flush interval")
	flag.DurationVar(&Options.BadClientPunishDuration, "punish", time.Second*3, "punish bad client by sleep")
	flag.DurationVar(&Options.MetaRefresh, "metarefresh", time.Minute*5, "meta data refresh interval")
	flag.DurationVar(&Options.ManagerRefresh, "manrefresh", time.Minute*5, "manager integration refresh interval")
	flag.DurationVar(&Options.PubPoolIdleTimeout, "pubpoolidle", 0, "pub pool connect idle timeout")
	flag.DurationVar(&Options.InternalServerErrorBackoff, "500backoff", time.Second, "internal server error backoff duration")

	flag.Parse()
}