예제 #1
0
파일: mirror.go 프로젝트: funkygao/gafka
func (this *Mirror) Main() (exitCode int) {
	this.quit = make(chan struct{})
	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("received signal: %s", strings.ToUpper(sig.String()))
		log.Info("quiting...")

		this.once.Do(func() {
			close(this.quit)
		})
	}, syscall.SIGINT, syscall.SIGTERM)

	limit := (1 << 20) * this.BandwidthLimit / 8
	if this.BandwidthLimit > 0 {
		this.bandwidthRateLimiter = ratelimiter.NewLeakyBucket(limit*10, time.Second*10)
	}

	log.Info("starting mirror@%s", gafka.BuildId)

	// pprof
	debugAddr := ":10009"
	go http.ListenAndServe(debugAddr, nil)
	log.Info("pprof ready on %s", debugAddr)

	z1 := zk.NewZkZone(zk.DefaultConfig(this.Z1, ctx.ZoneZkAddrs(this.Z1)))
	z2 := zk.NewZkZone(zk.DefaultConfig(this.Z2, ctx.ZoneZkAddrs(this.Z2)))
	c1 := z1.NewCluster(this.C1)
	c2 := z2.NewCluster(this.C2)

	this.runMirror(c1, c2, limit)

	log.Info("bye mirror@%s, %s", gafka.BuildId, time.Since(this.startedAt))
	log.Close()

	return
}
예제 #2
0
파일: mirror.go 프로젝트: chendx79/gafka
func (this *Mirror) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("mirror", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone1, "z1", "", "")
	cmdFlags.StringVar(&this.zone2, "z2", "", "")
	cmdFlags.StringVar(&this.cluster1, "c1", "", "")
	cmdFlags.StringVar(&this.cluster2, "c2", "", "")
	cmdFlags.StringVar(&this.excludes, "excluded", "", "")
	cmdFlags.BoolVar(&this.debug, "debug", false, "")
	cmdFlags.StringVar(&this.compress, "compress", "", "")
	cmdFlags.Int64Var(&this.bandwidthLimit, "net", 100, "")
	cmdFlags.BoolVar(&this.autoCommit, "commit", true, "")
	cmdFlags.Int64Var(&this.progressStep, "step", 5000, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z1", "-z2", "-c1", "-c2").
		invalid(args) {
		return 2
	}

	this.topicsExcluded = make(map[string]struct{})
	for _, e := range strings.Split(this.excludes, ",") {
		this.topicsExcluded[e] = struct{}{}
	}

	log.SetOutput(os.Stdout)
	this.quit = make(chan struct{})
	limit := (1 << 20) * this.bandwidthLimit / 8
	this.bandwidthRateLimiter = ratelimiter.NewLeakyBucket(limit, time.Second)
	log.Printf("[%s]%s -> [%s]%s with bandwidth %sbps",
		this.zone1, this.cluster1,
		this.zone2, this.cluster2,
		gofmt.Comma(int64(limit*8)))
	signal.RegisterSignalsHandler(func(sig os.Signal) {
		log.Printf("received signal: %s", strings.ToUpper(sig.String()))
		log.Println("quiting...")

		this.once.Do(func() {
			close(this.quit)
		})
	}, syscall.SIGINT, syscall.SIGTERM)

	z1 := zk.NewZkZone(zk.DefaultConfig(this.zone1, ctx.ZoneZkAddrs(this.zone1)))
	z2 := zk.NewZkZone(zk.DefaultConfig(this.zone2, ctx.ZoneZkAddrs(this.zone2)))
	c1 := z1.NewCluster(this.cluster1)
	c2 := z2.NewCluster(this.cluster2)
	this.makeMirror(c1, c2)

	return
}
예제 #3
0
파일: stat.go 프로젝트: funkygao/gafka
func (this *Stat) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("stat", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.zone == "" {
		this.Ui.Error("unknown zone")
		return 2
	}

	if len(args) == 0 {
		this.Ui.Error("missing path")
		return 2
	}

	this.path = args[len(args)-1]

	zkzone := gzk.NewZkZone(gzk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	defer zkzone.Close()
	conn := zkzone.Conn()

	_, stat, err := conn.Get(this.path)
	must(err)
	this.Ui.Output(fmt.Sprintf("%# v", pretty.Formatter(*stat)))
	ctime := gzk.ZkTimestamp(stat.Ctime).Time()
	mtime := gzk.ZkTimestamp(stat.Mtime).Time()
	this.Ui.Output(fmt.Sprintf("ctime: %s, mtime: %s",
		gofmt.PrettySince(ctime), gofmt.PrettySince(mtime)))
	return
}
예제 #4
0
파일: partition.go 프로젝트: chendx79/gafka
func (this *Partition) Run(args []string) (exitCode int) {
	var (
		zone       string
		topic      string
		cluster    string
		partitions int
	)
	cmdFlags := flag.NewFlagSet("partition", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.StringVar(&topic, "t", "", "")
	cmdFlags.IntVar(&partitions, "n", 1, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z", "-c", "-t", "-n").
		requireAdminRights("-z").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	zkcluster := zkzone.NewCluster(cluster)
	this.addPartition(zkcluster.ZkConnectAddr(), topic, partitions)
	return
}
예제 #5
0
파일: topology.go 프로젝트: funkygao/gafka
func (this *Topology) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("topology", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.hostPattern, "host", "", "")
	cmdFlags.BoolVar(&this.verbose, "l", false, "")
	cmdFlags.BoolVar(&this.watchMode, "w", false, "")
	cmdFlags.BoolVar(&this.maxPort, "maxport", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	for {
		if this.maxPort {
			this.displayZoneMaxPort(zkzone)
		} else {
			this.displayZoneTopology(zkzone)
		}

		if !this.watchMode {
			return
		}
		time.Sleep(time.Second * 5)
		refreshScreen()
	}

	return
}
예제 #6
0
파일: job.go 프로젝트: funkygao/gafka
func (this *Job) Run(args []string) (exitCode int) {
	var (
		zone    string
		appid   string
		initJob string
	)
	cmdFlags := flag.NewFlagSet("job", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.DefaultZone(), "")
	cmdFlags.StringVar(&appid, "app", "", "")
	cmdFlags.IntVar(&this.due, "d", 0, "")
	cmdFlags.StringVar(&initJob, "init", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if initJob != "" {
		this.initializeJob(initJob)
		return
	}

	if appid != "" {
		this.displayAppJobs(appid)
		return
	}

	this.printResourcesAndActors()

	return
}
예제 #7
0
파일: kguard.go 프로젝트: funkygao/gafka
func (this *Kguard) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("kguard", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&this.longFmt, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	kguards, err := zkzone.KguardInfos()
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %v", zk.KguardLeaderPath, err.Error()))
		return
	}

	leader := kguards[0]
	this.Ui.Output(fmt.Sprintf("%s(out of %d candidates) up: %s",
		color.Green(leader.Host), leader.Candidates,
		gofmt.PrettySince(leader.Ctime)))

	if this.longFmt {
		this.showKguardVersion(leader.Host)
		this.showStats(leader.Host)
	}

	return
}
예제 #8
0
func (this *Controllers) Run(args []string) (exitCode int) {
	var (
		zone string
	)
	cmdFlags := flag.NewFlagSet("controllers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "") // TODO not used
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.printControllers(zkzone)
			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.printControllers(zkzone)
	printSwallowedErrors(this.Ui, zkzone)

	return
}
예제 #9
0
파일: create.go 프로젝트: chendx79/gafka
func (this *Create) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("create", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.path, "p", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-p").
		requireAdminRights("-p").
		invalid(args) {
		return 2
	}

	if this.zone == "" {
		this.Ui.Error("unknown zone")
		return 2
	}

	zkzone := gzk.NewZkZone(gzk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	defer zkzone.Close()
	conn := zkzone.Conn()

	data := inData()
	flags := int32(0)
	acl := zk.WorldACL(zk.PermAll)
	_, err := conn.Create(this.path, data, flags, acl)
	must(err)

	return
}
예제 #10
0
파일: rebalance.go 프로젝트: funkygao/gafka
func (this *Rebalance) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("rebalance", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.topic, "t", "", "")
	cmdFlags.StringVar(&this.partition, "p", "", "comma seperated ids")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z", "-c", "-t", "-p").
		requireAdminRights("-t").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.zkcluster = zkzone.NewCluster(this.cluster)

	data := this.generateReassignFile()
	this.Ui.Output(data)
	yes, _ := this.Ui.Ask("Are you sure to execute? [Y/N]")
	if yes == "Y" {
		this.executeReassignment()
	} else {
		this.Ui.Output("bye")
	}

	return
}
예제 #11
0
파일: ping.go 프로젝트: chendx79/gafka
// TODO run 3 nodes in a zone to monitor as daemon
// register the 3 nodes as host service tag.
func (this *Ping) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("ping", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.DurationVar(&this.interval, "interval", time.Minute*5, "")
	cmdFlags.StringVar(&this.logfile, "logfile", "stdout", "")
	cmdFlags.BoolVar(&this.problematicMode, "p", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.setupLog()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))

	for {
		this.diagnose()
		if this.logfile == "stdout" {
			break
		}

		time.Sleep(this.interval)
	}

	return
}
예제 #12
0
파일: consumers.go 프로젝트: funkygao/gafka
func (this *Consumers) Run(args []string) (exitCode int) {
	var (
		cluster string
		zone    string
	)
	cmdFlags := flag.NewFlagSet("consumers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.StringVar(&this.groupPattern, "g", "", "")
	cmdFlags.BoolVar(&this.onlineOnly, "online", false, "")
	cmdFlags.BoolVar(&this.byHost, "byhost", false, "")
	cmdFlags.StringVar(&this.topicPattern, "t", "", "")
	cmdFlags.BoolVar(&this.warnOnly, "warn", false, "")
	cmdFlags.BoolVar(&this.ownerOnly, "own", false, "")
	cmdFlags.BoolVar(&this.cleanup, "cleanup", false, "")
	cmdFlags.BoolVar(&this.confirmYes, "yes", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		requireAdminRights("-cleanup").
		invalid(args) {
		return 2
	}

	if this.ownerOnly {
		this.onlineOnly = true
	}

	if zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			switch {
			case this.cleanup:
				this.cleanupStaleConsumerGroups(zkzone, cluster)
			case this.byHost:
				this.printConsumersByHost(zkzone, cluster)
			default:
				this.printConsumersByGroupTable(zkzone, cluster)
			}
		})

		return
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	switch {
	case this.cleanup:
		this.cleanupStaleConsumerGroups(zkzone, cluster)
	case this.byHost:
		this.printConsumersByHost(zkzone, cluster)
	default:
		this.printConsumersByGroupTable(zkzone, cluster)
	}

	return
}
예제 #13
0
파일: zk.go 프로젝트: chendx79/gafka
func New(zone string, id string, data []byte) *zkreg {
	this := &zkreg{
		zkzone:     zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))),
		id:         id,
		data:       data,
		shutdownCh: make(chan struct{}),
	}

	return this
}
예제 #14
0
파일: migrate.go 프로젝트: funkygao/gafka
func (this *Migrate) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("migrate", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.topic, "t", "", "")
	cmdFlags.StringVar(&this.partition, "p", "", "")
	cmdFlags.StringVar(&this.brokerId, "brokers", "", "")
	cmdFlags.BoolVar(&this.verifyMode, "verify", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z", "-c", "-t").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.zkcluster = zkzone.NewCluster(this.cluster)

	if this.verifyMode {
		this.Ui.Info(fmt.Sprintf("You MUST manually remove the %s after migration is done.", reassignNodeFilename))
		this.Ui.Info(fmt.Sprintf("After verify ok, modify producer/consumer to point to new brokers!"))

		for {
			this.Ui.Output(fmt.Sprintf("%s", time.Now().String()))
			this.verify()

			time.Sleep(time.Second * 5)
		}

		return
	}

	if validateArgs(this, this.Ui).
		require("-z", "-c", "-t", "-p", "-brokers").
		requireAdminRights("-z").
		invalid(args) {
		return 2
	}

	//this.ensureBrokersAreAlive()
	data := this.generateReassignFile()
	this.Ui.Output(data)
	yes, _ := this.Ui.Ask("Are you sure to execute the migration? [Y/N]")
	if yes == "Y" {
		this.executeReassignment()
	} else {
		this.Ui.Output("bye")
	}

	return
}
예제 #15
0
파일: utils.go 프로젝트: chendx79/gafka
func forAllSortedZones(fn func(zkzone *zk.ZkZone)) {
	for _, zone := range ctx.SortedZones() {
		zkAddrs := ctx.ZoneZkAddrs(zone)
		if strings.TrimSpace(zkAddrs) == "" {
			continue
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, zkAddrs))
		fn(zkzone)
	}
}
예제 #16
0
파일: fs.go 프로젝트: funkygao/gafka
func New(zone, cluster string) *KafkaFS {
	log.Info("starting kfs zone:%s, cluster:%s", zone, cluster)

	this := &KafkaFS{}

	ctx.LoadFromHome()
	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.zkcluster = zkzone.NewCluster(cluster) // panic if invalid cluster

	this.root = this.newDir(os.FileMode(0777))
	return this
}
예제 #17
0
파일: start.go 프로젝트: funkygao/gafka
func (this *Start) main() {
	ctx.LoadFromHome()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkConnEvt, ok := this.zkzone.SessionEvents()
	if !ok {
		panic("someone stealing my events")
	}

	registry.Default = zkr.New(this.zkzone)

	log.Info("ehaproxy[%s] starting...", gafka.BuildId)
	go this.runMonitorServer(this.httpAddr)

	zkConnected := false
	for {
		instances, instancesChange, err := registry.Default.WatchInstances()
		if err != nil {
			log.Error("zone[%s] %s", this.zkzone.Name(), err)
			time.Sleep(time.Second)
			continue
		}

		if zkConnected {
			if len(instances) > 0 {
				this.reload(instances)
			} else {
				// resilience to zk problem by local cache
				log.Warn("backend all shutdown? skip this change")
				time.Sleep(time.Second)
				continue
			}
		}

		select {
		case <-this.quitCh:
			return

		case evt := <-zkConnEvt:
			if evt.State == zklib.StateHasSession && !zkConnected {
				log.Info("zk connected")
				zkConnected = true
			} else if zkConnected {
				log.Warn("zk jitter: %+v", evt)
			}

		case <-instancesChange:
			log.Info("instances changed!!")
		}
	}

}
예제 #18
0
파일: zk_test.go 프로젝트: funkygao/gafka
func TestAll(t *testing.T) {
	zkzone := zk.NewZkZone(zk.DefaultConfig(ctx.DefaultZone(), ctx.ZoneZkAddrs(ctx.DefaultZone())))
	defer zkzone.Close()
	z := New(DefaultConfig(), zkzone)
	z.Start()

	assert.Equal(t, "/kafka_pubsub", z.ZkChroot("me"))
	assert.Equal(t, []string{"localhost:2181"}, z.ZkAddrs())

	t.Logf("%+v", z.BrokerList("me"))
	t.Logf("%+v", z.TopicPartitions("me", "app1.foobar.v1"))
	t.Logf("%+v", z.ZkCluster("me"))
	n, err := z.OnlineConsumersCount("me", "app1.foobar.v1", "group")
	t.Logf("%d %+v", n, err)
}
예제 #19
0
파일: zk_test.go 프로젝트: funkygao/gafka
func TestZkPath(t *testing.T) {
	zkzone := zk.NewZkZone(zk.DefaultConfig(ctx.DefaultZone(), ctx.ZoneZkAddrs(ctx.DefaultZone())))
	defer zkzone.Close()
	zk := New(zkzone)
	id := "1"
	assert.Equal(t, "/_kateway/ids/local/1", zk.mypath(id))

	data := []byte("foo, bar")
	err := zk.Register(id, data)
	assert.Equal(t, nil, err)
	defer zk.Deregister(id, data)

	ok, err := zk.Registered(id)
	assert.Equal(t, true, ok)
	assert.Equal(t, nil, err)
}
예제 #20
0
파일: utils.go 프로젝트: chendx79/gafka
func forSortedZones(fn func(zkzone *zk.ZkZone)) {
	for _, zone := range ctx.SortedZones() {
		if strings.HasPrefix(zone, "z_") {
			// zk only
			continue
		}

		zkAddrs := ctx.ZoneZkAddrs(zone)
		if strings.TrimSpace(zkAddrs) == "" {
			continue
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, zkAddrs))
		fn(zkzone)
	}
}
예제 #21
0
파일: offset.go 프로젝트: funkygao/gafka
func (this *Offset) Run(args []string) (exitCode int) {
	var (
		zone      string
		cluster   string
		topic     string
		group     string
		partition string
		offset    int64
	)
	cmdFlags := flag.NewFlagSet("offset", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.StringVar(&topic, "t", "", "")
	cmdFlags.StringVar(&group, "g", "", "")
	cmdFlags.Int64Var(&offset, "offset", -1, "")
	cmdFlags.StringVar(&partition, "p", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-z", "-c", "-t", "-g", "-p", "-offset").
		requireAdminRights("-z").
		invalid(args) {
		return 2
	}

	if offset < 0 {
		this.Ui.Error("offset must be positive")
		return
	}

	if p, err := strconv.Atoi(partition); err != nil {
		this.Ui.Error("invalid partition")
		return
	} else if p < 0 || p > 100 {
		this.Ui.Error("invalid partition")
		return
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	zkcluster := zkzone.NewCluster(cluster)
	zkcluster.ResetConsumerGroupOffset(topic, group, partition, offset)
	this.Ui.Output("done")
	return
}
예제 #22
0
파일: db.go 프로젝트: chendx79/gafka
func New(cf *config) *mysqlStore {
	if cf == nil || cf.Zone == "" {
		panic("empty zone")
	}
	zkAddrs := ctx.ZoneZkAddrs(cf.Zone)
	if len(zkAddrs) == 0 {
		panic("empty zookeeper addr")
	}

	return &mysqlStore{
		cf:                     cf,
		zkzone:                 zk.NewZkZone(zk.DefaultConfig(cf.Zone, zkAddrs)), // TODO session timeout
		shutdownCh:             make(chan struct{}),
		refreshCh:              make(chan struct{}),
		allowUnregisteredGroup: false,
	}
}
예제 #23
0
파일: rm.go 프로젝트: chendx79/gafka
func (this *Rm) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("rm", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.path, "p", "", "")
	cmdFlags.BoolVar(&this.recursive, "R", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		require("-p").
		requireAdminRights("-p").
		invalid(args) {
		return 2
	}

	if this.zone == "" {
		this.Ui.Error("unknown zone")
		return 2
	}

	zkzone := gzk.NewZkZone(gzk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	defer zkzone.Close()
	if this.recursive {
		err := zkzone.DeleteRecursive(this.path)
		if err != nil {
			this.Ui.Error(err.Error())
		} else {
			this.Ui.Info(fmt.Sprintf("%s deleted ok", this.path))
		}

		return 0
	}

	// remove a single znode
	conn := zkzone.Conn()
	err := conn.Delete(this.path, -1)
	if err != nil {
		this.Ui.Error(err.Error())
	} else {
		this.Ui.Info(fmt.Sprintf("%s deleted ok", this.path))
	}

	return
}
예제 #24
0
파일: ls.go 프로젝트: funkygao/gafka
func (this *Ls) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("ls", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&this.recursive, "R", false, "")
	cmdFlags.StringVar(&this.likePattern, "like", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if len(args) == 0 {
		this.Ui.Error("missing path")
		return 2
	}

	this.path = args[len(args)-1]

	if this.zone == "" {
		this.Ui.Error("unknown zone")
		return 2
	}

	zkzone := gzk.NewZkZone(gzk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	defer zkzone.Close()
	conn := zkzone.Conn()

	if this.recursive {
		this.showChildrenRecursively(conn, this.path)
		return
	}

	children, _, err := conn.Children(this.path)
	must(err)
	sort.Strings(children)
	if this.path == "/" {
		this.path = ""
	}
	for _, child := range children {
		this.Ui.Output(this.path + "/" + child)
	}

	return
}
예제 #25
0
파일: brokers.go 프로젝트: chendx79/gafka
func (this *Brokers) Run(args []string) (exitCode int) {
	var (
		zone  string
		debug bool
	)
	cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&debug, "debug", false, "")
	cmdFlags.BoolVar(&this.ipInNumber, "n", false, "")
	cmdFlags.BoolVar(&this.staleOnly, "stale", false, "")
	cmdFlags.BoolVar(&this.showVersions, "versions", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.showVersions {
		this.doShowVersions()
		return
	}

	if debug {
		sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags)
	}

	if zone != "" {
		ensureZoneValid(zone)

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		this.displayZoneBrokers(zkzone)

		return
	}

	// print all brokers on all zones by default
	forSortedZones(func(zkzone *zk.ZkZone) {
		this.displayZoneBrokers(zkzone)
	})

	return
}
예제 #26
0
func BenchmarkDirectKafkaProduce1K(b *testing.B) {
	msgSize := 1 << 10
	b.ReportAllocs()
	b.SetBytes(int64(msgSize))

	ctx.LoadFromHome()

	cf := zkmeta.DefaultConfig()
	zkzone := zk.NewZkZone(zk.DefaultConfig("local", ctx.ZoneZkAddrs("local")))
	cf.Refresh = time.Hour
	meta.Default = zkmeta.New(cf, zkzone)
	meta.Default.Start()
	store.DefaultPubStore = kafka.NewPubStore(100, 0, false, false, true)
	store.DefaultPubStore.Start()

	data := []byte(strings.Repeat("X", msgSize))
	for i := 0; i < b.N; i++ {
		store.DefaultPubStore.SyncPub("me", "foobar", nil, data)
	}
}
예제 #27
0
파일: lszk.go 프로젝트: funkygao/gafka
func (this *LsZk) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("lszk", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&this.recursive, "R", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	if this.cluster != "" {
		zkcluster := zkzone.NewCluster(this.cluster)
		this.printCluster(zkcluster)
	} else {
		zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
			this.printCluster(zkcluster)
		})
	}

	return
}
예제 #28
0
파일: monitor.go 프로젝트: funkygao/gafka
func (this *Monitor) Init() {
	var logFile, zone string
	flag.StringVar(&logFile, "log", "stdout", "log filename")
	flag.StringVar(&zone, "z", "", "zone, required")
	flag.StringVar(&this.apiAddr, "http", ":10025", "api http server addr")
	flag.StringVar(&this.influxdbAddr, "influxAddr", "", "influxdb addr, required")
	flag.StringVar(&this.influxdbDbName, "db", "", "influxdb db name, required")
	flag.StringVar(&this.externalDir, "confd", "", "external script config dir")
	flag.Parse()

	if zone == "" || this.influxdbDbName == "" || this.influxdbAddr == "" {
		panic("zone or influxdb empty, run help ")
	}

	ctx.LoadFromHome()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.watchers = make([]Watcher, 0, 10)
	this.quit = make(chan struct{})

	// export RESTful api
	this.setupRoutes()

	if logFile == "stdout" {
		log.AddFilter("stdout", log.TRACE, log.NewConsoleLogWriter())
	} else {
		log.DeleteFilter("stdout")

		filer := log.NewFileLogWriter(logFile, true, false, 0644)
		filer.SetRotateDaily(true)
		filer.SetFormat("[%d %T] [%L] (%S) %M")
		log.AddFilter("file", log.TRACE, filer)
	}

	rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute)
	if err != nil {
		panic(err)
	}
	telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
}
예제 #29
0
파일: discover.go 프로젝트: funkygao/gafka
// TODO cluster info will contain desciption,owner,etc.
func (this *Discover) Run(args []string) (exitCode int) {
	var (
		zone string
	)
	cmdFlags := flag.NewFlagSet("discover", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if zone != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		this.discoverClusters(zkzone)
	} else {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.discoverClusters(zkzone)
		})
	}

	return
}
예제 #30
0
파일: topbroker.go 프로젝트: chendx79/gafka
func (this *TopBroker) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("topbroker", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.topic, "t", "", "")
	cmdFlags.BoolVar(&this.shortIp, "shortip", false, "")
	cmdFlags.DurationVar(&this.interval, "i", time.Second*5, "refresh interval")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.signalsCh = make(map[string]chan struct{})
	this.hostOffsetCh = make(chan map[string]int64)

	this.offsets = make(map[string]int64)
	this.lastOffsets = make(map[string]int64)

	if this.interval.Seconds() < 1 {
		this.interval = time.Second
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), this.cluster) {
			return
		}

		this.signalsCh[zkcluster.Name()] = make(chan struct{})

		go this.clusterTopProducers(zkcluster)
	})

	this.drawDashboard()

	return
}