예제 #1
0
// Print all controllers of all clusters within a zone.
func (this *Controllers) printControllers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedControllers(func(cluster string, controller *zk.ControllerMeta) {
		if !patternMatched(cluster, this.cluster) {
			return
		}

		this.Ui.Output(strings.Repeat(" ", 4) + cluster)
		if controller == nil {
			this.Ui.Output(fmt.Sprintf("\t%s", color.Red("empty")))
		} else {
			epochSince := time.Since(controller.Mtime.Time())
			epochSinceStr := gofmt.PrettySince(controller.Mtime.Time())
			if epochSince < time.Hour*2*24 {
				epochSinceStr = color.Red(epochSinceStr)
			}
			this.Ui.Output(fmt.Sprintf("\t%-2s %21s epoch:%2s/%-20s uptime:%s",
				controller.Broker.Id, controller.Broker.Addr(),
				controller.Epoch,
				epochSinceStr,
				gofmt.PrettySince(controller.Broker.Uptime())))
		}
	})

}
예제 #2
0
파일: ping.go 프로젝트: chendx79/gafka
func (this *Ping) diagnose() {
	this.zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		registeredBrokers := zkcluster.RegisteredInfo().Roster
		for _, broker := range registeredBrokers {
			log.Debug("ping %s", broker.Addr())

			kfk, err := sarama.NewClient([]string{broker.Addr()}, sarama.NewConfig())
			if err != nil {
				log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))

				continue
			}

			_, err = kfk.Topics() // kafka didn't provide ping, so use Topics() as ping
			if err != nil {
				log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))
			} else {
				if !this.problematicMode {
					log.Info("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Green("ok"))
				}
			}
			kfk.Close()
		}
	})

}
예제 #3
0
파일: kateway.go 프로젝트: funkygao/gafka
func (this *Kateway) installGuide(zkzone *zk.ZkZone) {
	this.Ui.Output(color.Red("manager db GRANT access rights to this ip"))
	this.Ui.Output(color.Red("gk deploy -kfkonly"))
	this.Ui.Output("")

	this.Ui.Output("mkdir -p /var/wd/kateway/sbin")
	this.Ui.Output("cd /var/wd/kateway")
	kateways, err := zkzone.KatewayInfos()
	swallow(err)
	nextId := 1
	for _, kw := range kateways {
		id, _ := strconv.Atoi(kw.Id)
		if nextId < id {
			nextId = id
		}
	}
	nextId++

	zone := ctx.Zone(this.zone)
	influxAddr := zone.InfluxAddr
	if influxAddr != "" && !strings.HasPrefix(influxAddr, "http://") {
		influxAddr = "http://" + influxAddr
	}
	var influxInfo string
	if influxAddr != "" {
		influxInfo = "-influxdbaddr " + influxAddr
	}

	this.Ui.Output(fmt.Sprintf(`nohup ./sbin/kateway -zone prod -id %d -debughttp ":10194" -level trace -log kateway.log -crashlog panic %s &`,
		nextId, influxInfo))
	this.Ui.Output("")

	this.Ui.Output("yum install -y logstash")
	this.Ui.Output("/etc/logstash/conf.d/kateway.conf")
	this.Ui.Output(strings.TrimSpace(fmt.Sprintf(`
input {
    file {
        path => "/var/wd/kateway/kateway.log"
        type => "kateway"
    }
    file {
        path => "/var/wd/kateway/panic"
        type => "kateway_panic"
    }
}

output {
    kafka {
        bootstrap_servers => "%s:11003,%s:11003"
        topic_id => "pubsub_log"
    }
}
		`, color.Red("k11003a.mycorp.kfk.com"), color.Red("k11003b.mycorp.kfk.com"))))
	this.Ui.Output("")

	this.Ui.Output("chkconfig --add logstash")
	this.Ui.Output("/etc/init.d/logstash start")
}
예제 #4
0
파일: utils.go 프로젝트: chendx79/gafka
func (this *argsRule) invalid(args []string) bool {
	argSet := make(map[string]struct{}, len(args))
	for _, arg := range args {
		argSet[arg] = struct{}{}
	}

	// required
	for _, req := range this.requires {
		if _, present := argSet[req]; !present {
			this.ui.Error(color.Red("%s required", req))
			this.ui.Output(this.cmd.Help())
			return true
		}
	}

	// conditions
	for when, requires := range this.conditions {
		if _, present := argSet[when]; present {
			for _, req := range requires {
				if _, found := argSet[req]; !found {
					this.ui.Error(color.Red("%s required when %s present",
						req, when))
					this.ui.Output(this.cmd.Help())
					return true
				}
			}
		}
	}

	// admin required
	adminAuthRequired := false
	for _, arg := range args {
		if _, present := this.adminRequires[arg]; present {
			adminAuthRequired = true
			break
		}
	}
	if adminAuthRequired {
		if pass := os.Getenv("GK_PASS"); Authenticator("", pass) {
			return false
		}

		pass, err := this.ui.AskSecret("password for admin(or GK_PASS): ")
		this.ui.Output("")
		if err != nil {
			this.ui.Error(err.Error())
			return true
		}
		if !Authenticator("", pass) {
			this.ui.Error("invalid admin password, bye!")
			return true
		}
	}

	return false
}
예제 #5
0
파일: debug.go 프로젝트: postfix/golib-1
func Debugf(format string, args ...interface{}) {
	if Debug {
		pc, file, line, ok := runtime.Caller(1)
		if !ok {
			file = "<?>"
			line = 0
		} else {
			if i := strings.LastIndex(file, "/"); i >= 0 {
				file = file[i+1:]
			}
		}
		fn := runtime.FuncForPC(pc).Name()
		fnparts := strings.Split(fn, "/")
		t := time.Now()
		hour, min, sec := t.Clock()
		nanosec := t.Nanosecond() / 1e3

		debugLock.Lock()
		fmt.Printf("DEBUG: [%02d:%02d:%02d.%04d] %s:%d(%s): %s\n",
			hour, min, sec, nanosec,
			file, line, color.Red(fnparts[len(fnparts)-1]),
			fmt.Sprintf(format, args...))
		debugLock.Unlock()
	}
}
예제 #6
0
파일: zktop.go 프로젝트: funkygao/gafka
func (this *Zktop) displayZoneTop(zkzone *zk.ZkZone) {
	if this.batchMode {
		this.Ui.Output(fmt.Sprintf("%s %s", zkzone.Name(), bjtime.NowBj()))
	} else {
		this.Ui.Output(color.Green(zkzone.Name()))
	}

	header := "VER             SERVER           PORT M  OUTST            RECVD             SENT CONNS  ZNODES LAT(MIN/AVG/MAX)"
	this.Ui.Output(header)

	stats := zkzone.RunZkFourLetterCommand("stat")
	sortedHosts := make([]string, 0, len(stats))
	for hp, _ := range stats {
		sortedHosts = append(sortedHosts, hp)
	}
	sort.Strings(sortedHosts)

	for _, hostPort := range sortedHosts {
		host, port, err := net.SplitHostPort(hostPort)
		if err != nil {
			panic(err)
		}

		stat := zk.ParseStatResult(stats[hostPort])
		if stat.Mode == "" {
			if this.batchMode {
				stat.Mode = "E"
			} else {
				stat.Mode = color.Red("E")
			}
		} else if stat.Mode == "L" && !this.batchMode {
			stat.Mode = color.Blue(stat.Mode)
		}
		var sentQps, recvQps int
		if lastRecv, present := this.lastRecvs[hostPort]; present {
			r1, _ := strconv.Atoi(stat.Received)
			r0, _ := strconv.Atoi(lastRecv)
			recvQps = (r1 - r0) / int(this.refreshInterval.Seconds())

			s1, _ := strconv.Atoi(stat.Sent)
			s0, _ := strconv.Atoi(this.lastSents[hostPort])
			sentQps = (s1 - s0) / int(this.refreshInterval.Seconds())
		}
		this.Ui.Output(fmt.Sprintf("%-15s %-15s %5s %1s %6s %16s %16s %5s %7s %s",
			stat.Version,                                 // 15
			host,                                         // 15
			port,                                         // 5
			stat.Mode,                                    // 1
			stat.Outstanding,                             // 6
			fmt.Sprintf("%s/%d", stat.Received, recvQps), // 16
			fmt.Sprintf("%s/%d", stat.Sent, sentQps),     // 16
			stat.Connections,                             // 5
			stat.Znodes,                                  // 7
			stat.Latency,
		))

		this.lastRecvs[hostPort] = stat.Received
		this.lastSents[hostPort] = stat.Sent
	}
}
예제 #7
0
파일: table.go 프로젝트: funkygao/pastry
func (t *routingTable) debug(format string, v ...interface{}) {
	if t.logLevel <= LogLevelDebug {
		pc, file, line, ok := runtime.Caller(1)
		if !ok {
			file = "<?>"
			line = 0
		} else {
			if i := strings.LastIndex(file, "/"); i >= 0 {
				file = file[i+1:]
			}
		}
		fn := runtime.FuncForPC(pc).Name()
		fnparts := strings.Split(fn, "/")
		t1 := time.Now()
		hour, min, sec := t1.Clock()
		nanosec := t1.Nanosecond() / 1e3

		debugLock.Lock()

		fmt.Printf(t.self.ID.String()+" [%d:%d:%d.%04d] %s:%d(%s): %s\n",
			hour, min, sec, nanosec,
			file, line, color.Red(fnparts[len(fnparts)-1]),
			fmt.Sprintf(format, v...))
		debugLock.Unlock()
	}
}
예제 #8
0
파일: brokers.go 프로젝트: chendx79/gafka
func (this *Brokers) clusterBrokers(zone, cluster string, brokers map[string]*zk.BrokerZnode) []string {
	if !patternMatched(cluster, this.cluster) {
		return nil
	}

	if brokers == nil || len(brokers) == 0 {
		return []string{fmt.Sprintf("%s|%s|%s|%s|%s",
			zone, cluster, " ", color.Red("empty brokers"), " ")}
	}

	lines := make([]string, 0, len(brokers))
	if this.staleOnly {
		// try each broker's aliveness
		for brokerId, broker := range brokers {
			cf := sarama.NewConfig()
			cf.Net.ReadTimeout = time.Second * 4
			cf.Net.WriteTimeout = time.Second * 4
			kfk, err := sarama.NewClient([]string{broker.Addr()}, cf)
			if err != nil {
				lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
					zone, cluster,
					brokerId, broker.Addr(),
					fmt.Sprintf("%s: %v", gofmt.PrettySince(broker.Uptime()), err)))
			} else {
				kfk.Close()
			}
		}

		return lines
	}

	// sort by broker id
	sortedBrokerIds := make([]string, 0, len(brokers))
	for brokerId, _ := range brokers {
		sortedBrokerIds = append(sortedBrokerIds, brokerId)
	}
	sort.Strings(sortedBrokerIds)

	for _, brokerId := range sortedBrokerIds {
		b := brokers[brokerId]
		uptime := gofmt.PrettySince(b.Uptime())
		if time.Since(b.Uptime()) < time.Hour*24*7 {
			uptime = color.Green(uptime)
		}
		if this.ipInNumber {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.Addr(),
				gofmt.PrettySince(b.Uptime())))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.NamedAddr(),
				gofmt.PrettySince(b.Uptime())))
		}

	}
	return lines
}
예제 #9
0
파일: utils.go 프로젝트: chendx79/gafka
func printSwallowedErrors(ui cli.Ui, zkzone *zk.ZkZone) {
	errs := zkzone.Errors()
	if len(errs) == 0 {
		return
	}

	for _, e := range errs {
		ui.Error(color.Red("%v", e))
	}
}
예제 #10
0
파일: cluster.go 프로젝트: funkygao/pastry
func (c *Cluster) debug(format string, v ...interface{}) {
	if c.logLevel <= LogLevelDebug {
		pc, file, line, ok := runtime.Caller(1)
		if !ok {
			file = "<?>"
			line = 0
		} else {
			if i := strings.LastIndex(file, "/"); i >= 0 {
				file = file[i+1:]
			}
		}
		fn := runtime.FuncForPC(pc).Name()
		fnparts := strings.Split(fn, "/")
		t := time.Now()
		hour, min, sec := t.Clock()
		nanosec := t.Nanosecond() / 1e3

		debugLock.Lock()
		var nodePrefix string = c.self.ID.String()
		switch c.color {
		case "red":
			nodePrefix = color.Red(c.self.ID.String())
		case "blue":
			nodePrefix = color.Blue(c.self.ID.String())
		case "yellow":
			nodePrefix = color.Yellow(c.self.ID.String())
		case "green":
			nodePrefix = color.Green(c.self.ID.String())
		}
		fmt.Printf(nodePrefix+" [%d:%d:%d.%04d] %s:%d(%s): %s\n",
			hour, min, sec, nanosec,
			file, line, color.Red(fnparts[len(fnparts)-1]),
			fmt.Sprintf(format, v...))
		debugLock.Unlock()
	}
}
예제 #11
0
파일: flw.go 프로젝트: chendx79/gafka
// Returns {zkHost: outputLines}
func (this *ZkZone) RunZkFourLetterCommand(cmd string) map[string]string {
	servers := this.conf.ZkServers()
	r := make(map[string]string, len(servers))
	for _, server := range servers {
		b, err := zkFourLetterWord(server, cmd, time.Minute)
		if err != nil {
			r[server] = color.Red(err.Error())
		} else {
			r[server] = string(b)
		}

	}

	return r
}
예제 #12
0
파일: bury.go 프로젝트: funkygao/gafka
func sub(id int) {
	cf := api.DefaultConfig("app2", "mysecret")
	cf.Debug = true
	cf.Sub.Endpoint = addr
	c := api.NewClient(cf)
	i := 0
	t0 := time.Now()
	var err error
	opt := api.SubOption{
		AppId: appid,
		Topic: topic,
		Ver:   "v1",
		Group: group,
		Tag:   tag,
	}

	err = c.SubX(opt, func(statusCode int, msg []byte,
		r *api.SubXResult) error {
		i++
		if n > 0 && i >= n {
			return api.ErrSubStop
		}

		if i%step == 0 {
			log.Println(statusCode, string(msg))
		}

		if sleep > 0 {
			time.Sleep(sleep)
		}

		r.Bury = api.ShadowRetry
		log.Println(color.Red("shadow"))

		log.Println()

		return nil
	})

	if err != nil {
		log.Println(err)
	}

	elapsed := time.Since(t0)
	log.Printf("%d msgs in %s, tps: %.2f\n", n, elapsed, float64(n)/elapsed.Seconds())
}
예제 #13
0
파일: clusters.go 프로젝트: funkygao/gafka
func (this *Clusters) verifyBrokers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		zkcluster := zkzone.NewCluster(cluster)
		registeredBrokers := zkcluster.RegisteredInfo().Roster

		// find diff between registeredBrokers and liveBrokers
		// loop1 find liveBrokers>registeredBrokers
		for _, broker := range liveBrokers {
			foundInRoster := false
			for _, b := range registeredBrokers {
				bid := strconv.Itoa(b.Id)
				if bid == broker.Id && broker.Addr() == b.Addr() {
					foundInRoster = true
					break
				}
			}

			if !foundInRoster {
				// should manually register the broker
				this.Ui.Output(strings.Repeat(" ", 4) +
					color.Green("+ gk clusters -z %s -s -c %s -addbroker %s:%s",
						zkzone.Name(), cluster, broker.Id, broker.Addr()))
			}
		}

		// loop2 find liveBrokers<registeredBrokers
		for _, b := range registeredBrokers {
			foundInLive := false
			for _, broker := range liveBrokers {
				bid := strconv.Itoa(b.Id)
				if bid == broker.Id && broker.Addr() == b.Addr() {
					foundInLive = true
					break
				}
			}

			if !foundInLive {
				// the broker is dead
				this.Ui.Output(strings.Repeat(" ", 4) +
					color.Red("cluster[%s] broker[%d] %s is dead", cluster, b.Id, b.Addr()))
			}
		}
	})
}
예제 #14
0
파일: discover.go 프로젝트: funkygao/gafka
func (this *Discover) discoverClusters(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())

	existingClusters := zkzone.Clusters()
	existingCluserPaths := make(map[string]struct{}, len(existingClusters))
	for _, path := range existingClusters {
		existingCluserPaths[path] = struct{}{}
	}

	discoveredClusters, err := zkzone.DiscoverClusters("/")
	if err != nil {
		this.Ui.Error(zkzone.Name() + ": " + err.Error())
		return
	}

	// print each cluster state: new, normal
	for _, zkpath := range discoveredClusters {
		if _, present := existingCluserPaths[zkpath]; !present {
			this.Ui.Output(strings.Repeat(" ", 4) + color.Green("%s +++",
				zkpath))
		} else {
			this.Ui.Output(strings.Repeat(" ", 4) + zkpath)
		}
	}

	// find the offline clusters
	for c, path := range existingClusters {
		path = strings.TrimSpace(path)
		foundOnline := false
		for _, p := range discoveredClusters {
			p = strings.TrimSpace(p)
			if p == path {
				foundOnline = true
				break
			}
		}
		if !foundOnline {
			this.Ui.Output(strings.Repeat(" ", 4) + color.Red("%s: %s ---", c, path))
		}
	}
}
예제 #15
0
파일: verify.go 프로젝트: funkygao/gafka
func (this *Verify) verifyPub() {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Kafka", "Stock", "PubSub", "Stock", "Diff", "?"})
	for _, t := range this.topics {
		if t.KafkaTopicName == "" {
			continue
		}
		kafkaCluster := this.kafkaTopics[t.KafkaTopicName]
		if kafkaCluster == "" {
			this.Ui.Warn(fmt.Sprintf("invalid kafka topic: %s", t.KafkaTopicName))
			continue
		}

		psubTopic := manager.Default.KafkaTopic(t.AppId, t.TopicName, "v1")
		offsets := this.pubOffsetDiff(t.KafkaTopicName, kafkaCluster,
			psubTopic, this.cluster)
		var diff string
		if offsets[0] == 0 && offsets[1] != 0 {
			diff = color.Yellow("%d", offsets[1]-offsets[0])
		} else if math.Abs(float64(offsets[0]-offsets[1])) < 20 {
			diff = color.Green("%d", offsets[1]-offsets[0])
		} else {
			diff = color.Red("%d", offsets[1]-offsets[0])
		}

		problem := "N"
		if _, present := this.problemeticTopics[t.KafkaTopicName]; present {
			problem = color.Yellow("Y")
		}

		table.Append([]string{
			t.KafkaTopicName, fmt.Sprintf("%d", offsets[0]),
			t.TopicName, fmt.Sprintf("%d", offsets[1]), diff, problem})
	}

	table.Render()
}
예제 #16
0
파일: lags.go 프로젝트: chendx79/gafka
func (this *Lags) printConsumersLag(zkcluster *zk.ZkCluster) {
	// sort by group name
	consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern)
	sortedGroups := make([]string, 0, len(consumersByGroup))
	for group, _ := range consumersByGroup {
		sortedGroups = append(sortedGroups, group)
	}
	sort.Strings(sortedGroups)

	for _, group := range sortedGroups {
		lines := make([]string, 0, 100)

		sortedTopicAndPartitionIds := make([]string, 0)
		consumers := make(map[string]zk.ConsumerMeta)
		for _, t := range consumersByGroup[group] {
			key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId)
			sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key)

			consumers[key] = t
		}
		sort.Strings(sortedTopicAndPartitionIds)

		for _, topicAndPartitionId := range sortedTopicAndPartitionIds {
			consumer := consumers[topicAndPartitionId]

			if !patternMatched(consumer.Topic, this.topicPattern) {
				continue
			}

			var (
				lagOutput string
				symbol    string
			)
			if consumer.Lag > int64(this.lagThreshold) {
				lagOutput = color.Red("%15s", gofmt.Comma(consumer.Lag))
				if consumer.Online {
					symbol = color.Yellow("⚠︎︎")
				} else {
					symbol = color.Yellow("◎")
				}
			} else {
				lagOutput = color.Blue("%15s", gofmt.Comma(consumer.Lag))
				if consumer.Online {
					symbol = color.Green("◉")
				} else {
					symbol = color.Yellow("◎")
				}
			}

			if consumer.Online {
				if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) {
					continue
				}

				var (
					host   string
					uptime string
				)
				if consumer.ConsumerZnode == nil {
					host = "unrecognized"
					uptime = "-"
				} else {
					host = color.Green("%s", consumer.ConsumerZnode.Host())
					if time.Since(consumer.ConsumerZnode.Uptime()) < time.Hour {
						uptime = color.Magenta(gofmt.PrettySince(consumer.ConsumerZnode.Uptime()))
					} else {
						uptime = gofmt.PrettySince(consumer.ConsumerZnode.Uptime())
					}
				}

				lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-15s %s %-10s %s %s",
					symbol,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					lagOutput,
					gofmt.PrettySince(consumer.Mtime.Time()),
					host, uptime))
			} else if !this.onlineOnly {
				lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-12s %s %s",
					symbol,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					lagOutput,
					gofmt.PrettySince(consumer.Mtime.Time())))
			}
		}

		if len(lines) > 0 {
			this.Ui.Output(strings.Repeat(" ", 4) + group)
			for _, l := range lines {
				this.Ui.Output(l)
			}
		}
	}
}
예제 #17
0
파일: sample.go 프로젝트: chendx79/gafka
func (*Sample) consumeSample() string {
	return fmt.Sprintf(`
public class KafkaConsumer {
    private final ConsumerConnector consumer;

    private KafkaConsumer() {
        Properties props = new Properties();
        props.put("%s", "zk2181a.wdds.zk.com:2181,zk2181b.wdds.zk.com:2181,zk2181c.wdds.zk.com:2181/kafka");
        props.put("%s", "group1");
        props.put("zookeeper.session.timeout.ms", "4000");
        props.put("zookeeper.sync.time.ms", "200");
        props.put("auto.commit.interval.ms", "60000");   // 1m
        //props.put("auto.offset.reset", "smallest");    // largest | smallest
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        ConsumerConfig config = new ConsumerConfig(props);

        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config);
    }

    public void shutdown() {
        if (consumer != null) {
            consumer.shutdown();
        }
   }

    void consume(String topic, int %s) {
        // %s
        // %s
        // %s
        Runtime.getRuntime().addShutdownHook(new Thread() {
            public void run() {
                consumer.shutdown();
            }
        });

        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, %s);

        StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
        StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

        Map<String, List<KafkaStream<String, String>>> consumerMap = 
                consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
        KafkaStream<String, String> stream = consumerMap.get(topic).get(0);
        ConsumerIterator<String, String> it = stream.iterator();
        while (it.hasNext()) {
            // consumer.commitOffsets(); // manually commit offsets
            System.out.println(it.next().message());
        }
    }

    public static void main(String[] args) {
        new KafkaConsumer().consume();
    }
}   
        `,
		color.Cyan("zookeeper.connect"),
		color.Cyan("group.id"),
		color.Green("threads"),
		color.Red("VERY important!"),
		color.Red("graceful shutdown the consumer group to commit consumed offset"),
		color.Red("avoid consuming duplicated message when restarting the same consumer group"),
		color.Green("threads"))
}
예제 #18
0
파일: zkinstall.go 프로젝트: funkygao/gafka
func (this *ZkInstall) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("zkinstall", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.rootPath, "root", "/var/wd/zookeeper", "")
	cmdFlags.StringVar(&this.myId, "id", "", "")
	cmdFlags.StringVar(&this.servers, "servers", "", "")
	cmdFlags.BoolVar(&this.singleMode, "single", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if !ctx.CurrentUserIsRoot() {
		this.Ui.Error("requires root priviledges!")
		return 1
	}

	if !this.singleMode {
		if validateArgs(this, this.Ui).
			require("-id", "-servers").
			invalid(args) {
			return 2
		}
	}

	// create dirs
	this.rootPath = strings.TrimSuffix(this.rootPath, "/")
	for _, d := range []string{"bin", "conf", "data", "lib", "log"} {
		swallow(os.MkdirAll(fmt.Sprintf("%s/%s", this.rootPath, d), 0755))
	}

	type templateVar struct {
		MyId     string
		RootPath string
		Servers  string
	}
	data := templateVar{
		MyId:     this.myId,
		RootPath: this.rootPath,
	}
	if !this.singleMode {
		servers := make([]string, 0)
		for _, s := range strings.Split(this.servers, ",") {
			parts := strings.SplitN(s, ":", 2)
			servers = append(servers, fmt.Sprintf("server.%s=%s:2888:3888",
				parts[0], parts[1]))
		}
		data.Servers = strings.Join(servers, "\n")
	}

	// copy all files in bin and lib
	for srcDir, dstDir := range map[string]string{
		"template/zk/bin": fmt.Sprintf("%s/bin", this.rootPath),
		"template/zk/lib": fmt.Sprintf("%s/lib", this.rootPath)} {
		files, err := AssetDir(srcDir)
		swallow(err)
		for _, srcFile := range files {
			_, dstFile := path.Split(srcFile)
			from := fmt.Sprintf("%s/%s", srcDir, srcFile)
			to := fmt.Sprintf("%s/%s", dstDir, dstFile)
			var perm os.FileMode = 0644
			if strings.HasSuffix(srcDir, "/bin") {
				perm = 0755
			}
			writeFileFromTemplate(from, to, perm, nil, nil)
		}
	}

	// zk jar
	writeFileFromTemplate("template/zk/zookeeper-3.4.6.jar",
		fmt.Sprintf("%s/zookeeper-3.4.6.jar", this.rootPath), 0644, nil, nil)

	// tempated conf
	writeFileFromTemplate("template/zk/conf/zoo.cfg",
		fmt.Sprintf("%s/conf/zoo.cfg", this.rootPath), 0644, data, nil)
	writeFileFromTemplate("template/zk/conf/log4j.properties",
		fmt.Sprintf("%s/conf/log4j.properties", this.rootPath), 0644, nil, nil)

	// templated data/myid
	if !this.singleMode {
		writeFileFromTemplate("template/zk/data/myid",
			fmt.Sprintf("%s/data/myid", this.rootPath), 0644, data, nil)
	}

	// templated init.d/
	writeFileFromTemplate("template/init.d/zookeeper",
		"/etc/init.d/zookeeper", 0755, data, nil)

	this.Ui.Info("zookeeper installed on localhost")
	this.Ui.Warn(fmt.Sprintf("NOW, please run the following command:"))
	this.Ui.Warn("yum install -y jdk-1.7.0_65-fcs.x86_64")
	this.Ui.Output(color.Red("chkconfig --add zookeeper"))
	this.Ui.Output(color.Red("/etc/init.d/zookeeper start"))

	return
}
예제 #19
0
파일: kateway.go 프로젝트: funkygao/gafka
func (this *Kateway) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("kateway", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.BoolVar(&this.configMode, "cf", false, "")
	cmdFlags.StringVar(&this.id, "id", "", "")
	cmdFlags.BoolVar(&this.install, "i", false, "")
	cmdFlags.BoolVar(&this.longFmt, "l", false, "")
	cmdFlags.StringVar(&this.configOption, "option", "", "")
	cmdFlags.BoolVar(&this.versionOnly, "ver", false, "")
	cmdFlags.BoolVar(&this.flameGraph, "flame", false, "")
	cmdFlags.StringVar(&this.logLevel, "loglevel", "", "")
	cmdFlags.StringVar(&this.visualLog, "visualog", "", "")
	cmdFlags.BoolVar(&this.showZkNodes, "zk", false, "")
	cmdFlags.BoolVar(&this.checkup, "checkup", false, "")
	cmdFlags.BoolVar(&this.benchmark, "bench", false, "")
	cmdFlags.StringVar(&this.benchmarkMaster, "master", "", "")
	cmdFlags.BoolVar(&this.pub, "pub", false, "")
	cmdFlags.BoolVar(&this.sub, "sub", false, "")
	cmdFlags.BoolVar(&this.benchmarkAsync, "async", false, "")
	cmdFlags.BoolVar(&this.curl, "curl", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	if this.benchmark {
		if validateArgs(this, this.Ui).
			require("-z").
			requireAdminRights("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		zone := ctx.Zone(zkzone.Name())
		this.benchApp = zone.SmokeApp
		this.benchSecret = zone.SmokeSecret
		this.benchTopic = zone.SmokeTopic
		this.benchVer = zone.SmokeTopicVersion
		this.benchPubEndpoint = zone.PubEndpoint
		if this.id != "" {
			kws, err := zkzone.KatewayInfos()
			swallow(err)
			for _, kw := range kws {
				if kw.Id == this.id {
					this.benchPubEndpoint = kw.PubAddr
					break
				}
			}
		}
		this.benchId = fmt.Sprintf("%s-%s", ctx.Hostname(), strings.Replace(uuid.New(), "-", "", -1))
		this.runBenchmark(zkzone)
		return
	}

	if this.flameGraph {
		if validateArgs(this, this.Ui).
			require("-z", "-id").
			requireAdminRights("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.generateFlameGraph(zkzone)
		return
	}

	if this.visualLog != "" {
		this.doVisualize()
		return
	}

	if this.pub {
		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.runPub(zkzone)
		return
	}

	if this.sub {
		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.runSub(zkzone)
		return
	}

	if this.install {
		if validateArgs(this, this.Ui).
			require("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.installGuide(zkzone)
		return
	}

	if this.configOption != "" {
		this.configMode = true
	}

	if this.configMode {
		if validateArgs(this, this.Ui).
			require("-z").
			requireAdminRights("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		if this.logLevel != "" {
			if this.id != "" {
				kw := zkzone.KatewayInfoById(this.id)
				if kw == nil {
					panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id))
				}

				this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel))
			} else {
				// apply on all kateways
				kws, _ := zkzone.KatewayInfos()
				for _, kw := range kws {
					this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel))
				}
			}
		}

		if this.configOption != "" {
			parts := strings.SplitN(this.configOption, "=", 2)
			if len(parts) != 2 {
				this.Ui.Error("usage: key=value")
				return
			}
			k, v := parts[0], parts[1]
			if this.id != "" {
				kw := zkzone.KatewayInfoById(this.id)
				if kw == nil {
					panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id))
				}

				this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v))
			} else {
				// apply on all kateways
				kws, _ := zkzone.KatewayInfos()
				for _, kw := range kws {
					this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v))
				}
			}
		}

		return
	}

	if this.checkup {
		if this.zone == "" {
			forAllSortedZones(func(zkzone *zk.ZkZone) {
				this.runCheckup(zkzone)
			})

			return
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.runCheckup(zkzone)
		return
	}

	if this.showZkNodes {
		this.Ui.Output(fmt.Sprintf(`%s pubsub manager db dsn
%s job db cluster config
%s turn off webhook dir`,
			color.Green("%-50s", zk.KatewayMysqlPath),
			color.Green("%-50s", zk.PubsubJobConfig),
			color.Green("%-50s", zk.PubsubWebhooksOff)))
		return
	}

	// display mode
	lines := make([]string, 0)
	header := "Zone|Id|Ip|Pprof|Build|Cpu|Heap|Obj|Go|P/S|hhIn/hhOut|Uptime"
	lines = append(lines, header)
	forSortedZones(func(zkzone *zk.ZkZone) {
		if this.zone != "" && zkzone.Name() != this.zone {
			return
		}

		if !this.versionOnly {
			mysqlDsn, err := zkzone.KatewayMysqlDsn()
			if err != nil {
				this.Ui.Warn(fmt.Sprintf("kateway[%s] mysql DSN not set on zk yet", zkzone.Name()))
				this.Ui.Output(fmt.Sprintf("e,g. %s -> pubsub:pubsub@tcp(10.77.135.217:10010)/pubsub?charset=utf8&timeout=10s",
					zk.KatewayMysqlPath))
			} else {
				this.Ui.Output(fmt.Sprintf("zone[%s] manager db: %s",
					color.Cyan(zkzone.Name()), mysqlDsn))
			}
		}

		kateways, err := zkzone.KatewayInfos()
		if err != nil {
			if err == zklib.ErrNoNode {
				this.Ui.Output("no kateway running")
				return
			} else {
				swallow(err)
			}

		}

		for _, kw := range kateways {
			if this.id != "" && this.id != kw.Id {
				continue
			}

			statusMap, _ := this.getKatewayStatusMap(kw.ManAddr)
			logLevel, _ := statusMap["loglevel"].(string)
			heapSize, _ := statusMap["heap"].(string)
			heapObjs, _ := statusMap["objects"].(string)
			pubConn, _ := statusMap["pubconn"].(string)
			hhAppendN, _ := statusMap["hh_appends"].(string)
			hhDeliverN, _ := statusMap["hh_delivers"].(string)
			subConn, _ := statusMap["subconn"].(string)
			goN, _ := statusMap["goroutines"].(string)

			if this.versionOnly {
				pprofAddr := kw.DebugAddr
				if len(pprofAddr) > 0 && pprofAddr[0] == ':' {
					pprofAddr = kw.Ip + pprofAddr
				}
				pprofAddr = fmt.Sprintf("%s/debug/pprof/", pprofAddr)
				lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s/%s|%s|%s|%s|%s|%s/%s|%s/%s|%s",
					zkzone.Name(),
					kw.Id, kw.Ip,
					pprofAddr, kw.Build, kw.BuiltAt,
					kw.Cpu,
					heapSize, heapObjs,
					goN,
					pubConn, subConn, hhAppendN, hhDeliverN,
					gofmt.PrettySince(kw.Ctime)))
				continue
			}

			this.Ui.Info(fmt.Sprintf("id:%-2s host:%s cpu:%-2s up:%s",
				kw.Id, kw.Host, kw.Cpu,
				gofmt.PrettySince(kw.Ctime)))
			this.Ui.Output(fmt.Sprintf("    ver: %s\n   arch: %s\n  build: %s\n  built: %s\n    log: %s\n    pub: %s\n    sub: %s\n    man: %s\n    dbg: %s",
				kw.Ver,
				kw.Arch,
				color.Red(kw.Build),
				kw.BuiltAt,
				logLevel,
				kw.PubAddr,
				kw.SubAddr,
				kw.ManAddr,
				kw.DebugAddr,
			))

			if this.longFmt {
				this.Ui.Output("    full status:")
				this.Ui.Output(this.getKatewayStatus(kw.ManAddr))
			}

		}

	})

	if this.versionOnly && len(lines) > 1 {
		fmt.Println(columnize.SimpleFormat(lines))
	}

	return
}
예제 #20
0
파일: topology.go 프로젝트: funkygao/gafka
func (this *Topology) displayZoneTopology(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())

	// {cluster: {topic: brokerHostInfo}}
	brokerInstances := make(map[string]map[string]*brokerHostInfo)

	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		if len(liveBrokers) == 0 {
			this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
			return
		}
		if this.cluster != "" && this.cluster != cluster {
			return
		}

		brokerInstances[cluster] = make(map[string]*brokerHostInfo)

		for _, broker := range liveBrokers {
			if !patternMatched(broker.Host, this.hostPattern) {
				continue
			}

			if _, present := brokerInstances[cluster][broker.Host]; !present {
				brokerInstances[cluster][broker.Host] = newBrokerHostInfo()
			}
			brokerInstances[cluster][broker.Host].addPort(broker.Port, broker.Uptime())
		}

		// find how many partitions a broker is leading
		zkcluster := zkzone.NewCluster(cluster)
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
			return
		}

		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			this.Ui.Error(color.Red("    %+v %s", brokerList, err.Error()))
			return
		}

		topics, err := kfk.Topics()
		swallow(err)
		for _, topic := range topics {
			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)
				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)
				if !patternMatched(host, this.hostPattern) {
					continue
				}

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				if err != nil {
					this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
					continue
				}
				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				if err != nil {
					this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
					continue
				}

				brokerInstances[cluster][host].topicMsgs[topic] += (latestOffset - oldestOffset)
				brokerInstances[cluster][host].addTopicPartition(topic, partitionID)
			}
		}
	})

	hosts := make(map[string]struct{})
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for host, _ := range brokerInstances[zkcluster.Name()] {
			hosts[host] = struct{}{}
		}
	})
	sortedHosts := make([]string, 0)
	for host, _ := range hosts {
		sortedHosts = append(sortedHosts, host)
	}
	sort.Strings(sortedHosts)

	// sort by host ip
	sortedClusters := make([]string, 0, len(brokerInstances))
	for c, _ := range brokerInstances {
		sortedClusters = append(sortedClusters, c)
	}
	sort.Strings(sortedClusters)

	portN := 0
	hostN := 0
	topicN := 0
	partitionN := 0
	for _, host := range sortedHosts {
		tn := 0
		pn := 0
		mn := int64(0)
		ports := make([]int, 0)
		for _, cluster := range sortedClusters {
			if _, present := brokerInstances[cluster][host]; !present {
				continue
			}

			tn += len(brokerInstances[cluster][host].topicPartitions)
			pn += brokerInstances[cluster][host].leadingPartitions()
			mn += brokerInstances[cluster][host].totalMsgsInStock()
			ports = append(ports, brokerInstances[cluster][host].tcpPorts...)
		}

		portN += len(ports)
		topicN += tn
		partitionN += pn
		hostN += 1

		this.Ui.Output(fmt.Sprintf("  %s leading: %2dT %3dP %15sM ports %2d:%+v",
			color.Green("%15s", host),
			tn,
			pn,
			gofmt.Comma(mn),
			len(ports),
			ports))

		if this.verbose {
			for _, cluster := range sortedClusters {
				if _, present := brokerInstances[cluster][host]; !present {
					continue
				}

				for _, tcpPort := range brokerInstances[cluster][host].tcpPorts {
					this.Ui.Output(fmt.Sprintf("%40d %s", tcpPort,
						gofmt.PrettySince(brokerInstances[cluster][host].uptimes[tcpPort])))
				}
			}

			for _, cluster := range sortedClusters {
				if _, present := brokerInstances[cluster][host]; !present {
					continue
				}

				this.Ui.Output(color.Magenta("%30s", cluster))
				for topic, partitions := range brokerInstances[cluster][host].topicPartitions {
					this.Ui.Output(fmt.Sprintf("%40s: %15sM P%2d %+v",
						topic,
						gofmt.Comma(brokerInstances[cluster][host].topicMsgs[topic]),
						len(partitions), partitions))
				}

			}

		}
	}

	this.Ui.Output(fmt.Sprintf("%17s host:%d, topic:%d, partition:%d, instance:%d",
		"-TOTAL-",
		hostN, topicN, partitionN, portN))
}
예제 #21
0
파일: sub.go 프로젝트: funkygao/gafka
func sub(id int) {
	cf := api.DefaultConfig("app2", "mysecret")
	cf.Debug = true
	cf.Sub.Endpoint = addr
	c := api.NewClient(cf)
	i := 0
	t0 := time.Now()
	var err error
	opt := api.SubOption{
		AppId: appid,
		Topic: topic,
		Ver:   "v1",
		Group: group,
		Tag:   tag,
	}
	if mode == "subx" {
		err = c.SubX(opt, func(statusCode int, msg []byte,
			r *api.SubXResult) error {
			i++
			if n > 0 && i >= n {
				return api.ErrSubStop
			}

			if i%step == 0 {
				log.Println(statusCode, string(msg))
			}

			if sleep > 0 {
				time.Sleep(sleep)
			}

			// handle the msg here
			if rand.Int()%2 == 0 {
				// simulate handle this msg successfully
				log.Println(color.Green("ok"))
			} else {
				// this msg was not successfully handled
				if rand.Int()%2 == 0 {
					// after retry several times, give up
					r.Bury = api.ShadowRetry
					log.Println(color.Red("shadow"))
				} else {
					// simulate handle msg successfully after retry
					if sleep > 0 {
						time.Sleep(sleep)
					}
					log.Println(color.Yellow("retried"))
				}
			}

			log.Println()

			return nil
		})
	} else {
		err = c.Sub(opt, func(statusCode int, msg []byte) error {
			i++
			if n > 0 && i >= n {
				return api.ErrSubStop
			}

			if i%step == 0 {
				log.Println(id, statusCode, string(msg))
			}

			if sleep > 0 {
				time.Sleep(sleep)
			}

			return nil
		})
	}

	if err != nil {
		log.Println(err)
	}

	elapsed := time.Since(t0)
	log.Printf("%d msgs in %s, tps: %.2f\n", n, elapsed, float64(n)/elapsed.Seconds())
}
예제 #22
0
파일: deploy.go 프로젝트: funkygao/gafka
// TODO
// 1. broker id assignment
// 2. port assignment
func (this *Deploy) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("deploy", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.kafkaBaseDir, "kafka.base", ctx.KafkaHome(), "")
	cmdFlags.StringVar(&this.brokerId, "broker.id", "", "")
	cmdFlags.StringVar(&this.tcpPort, "port", "", "")
	cmdFlags.StringVar(&this.rootPah, "root", "/var/wd", "")
	cmdFlags.StringVar(&this.ip, "ip", "", "")
	cmdFlags.StringVar(&this.logDirs, "log.dirs", "", "")
	cmdFlags.StringVar(&this.runAs, "user", "sre", "")
	cmdFlags.StringVar(&this.uninstall, "uninstall", "", "")
	cmdFlags.BoolVar(&this.demoMode, "demo", false, "")
	cmdFlags.BoolVar(&this.installKafkaOnly, "kfkonly", false, "")
	cmdFlags.BoolVar(&this.dryRun, "dryrun", true, "")
	cmdFlags.StringVar(&this.influxDbAddr, "influx", "", "")
	cmdFlags.StringVar(&this.kafkaVer, "ver", "2.10-0.8.2.2", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.uninstall != "" {
		serverProperties := fmt.Sprintf("%s/config/server.properties", this.uninstall)
		lines, err := gio.ReadLines(serverProperties)
		if err != nil {
			this.Ui.Error(err.Error())
			return 2
		}
		var logDirs []string
		for _, line := range lines {
			if strings.HasPrefix(line, "log.dirs") {
				parts := strings.SplitN(line, "=", 2)
				logDirs = strings.Split(parts[1], ",")
				break
			}
		}
		if len(logDirs) == 0 {
			this.Ui.Error("empty log.dirs")
			return 2
		}

		for _, logDir := range logDirs {
			this.Ui.Output(fmt.Sprintf("rm -rf %s", logDir))
		}
		name := filepath.Base(this.uninstall)
		this.Ui.Output(fmt.Sprintf("chkconfig --del %s", name))
		this.Ui.Output(fmt.Sprintf("rm -f /etc/init.d/%s", name))
		this.Ui.Output(fmt.Sprintf("rm -rf %s", this.uninstall))
		return 0
	}

	if !ctx.CurrentUserIsRoot() {
		this.Ui.Error("requires root priviledges!")
		return 1
	}

	if !strings.HasSuffix(this.kafkaBaseDir, this.kafkaVer) {
		this.Ui.Error(fmt.Sprintf("kafka.base[%s] does not match ver[%s]",
			this.kafkaBaseDir, this.kafkaVer))
		return 1
	}

	if this.installKafkaOnly {
		this.installKafka()
		return
	}

	if validateArgs(this, this.Ui).
		require("-z", "-c").
		invalid(args) {
		return 2
	}

	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	clusers := this.zkzone.Clusters()
	zkchroot, present := clusers[this.cluster]
	if !present {
		this.Ui.Error(fmt.Sprintf("run 'gk clusters -z %s -add %s -p $zkchroot' first!",
			this.zone, this.cluster))
		return 1
	}

	var err error
	this.userInfo, err = user.Lookup(this.runAs)
	swallow(err)

	if this.demoMode {
		this.demo()
		return
	}

	if validateArgs(this, this.Ui).
		require("-broker.id", "-port", "-ip", "-log.dirs").
		invalid(args) {
		return 2
	}

	if this.dryRun {
		this.Ui.Output(fmt.Sprintf("mkdir %s/logs and chown to %s",
			this.instanceDir(), this.runAs))
	}
	err = os.MkdirAll(fmt.Sprintf("%s/logs", this.instanceDir()), 0755)
	swallow(err)
	chown(fmt.Sprintf("%s/logs", this.instanceDir()), this.userInfo)

	invalidDir := this.validateLogDirs(this.logDirs)
	if invalidDir != "" {
		this.Ui.Error(fmt.Sprintf("%s in log.dirs not exists!", invalidDir))
		return 2
	}

	// prepare the root directory
	this.rootPah = strings.TrimSuffix(this.rootPah, "/")
	if this.dryRun {
		this.Ui.Output(fmt.Sprintf("mkdir %s/bin and chown to %s",
			this.instanceDir(), this.runAs))
	}
	err = os.MkdirAll(fmt.Sprintf("%s/bin", this.instanceDir()), 0755)
	swallow(err)
	chown(fmt.Sprintf("%s/bin", this.instanceDir()), this.userInfo)

	if this.dryRun {
		this.Ui.Output(fmt.Sprintf("mkdir %s/config and chown to %s",
			this.instanceDir(), this.runAs))
	}
	err = os.MkdirAll(fmt.Sprintf("%s/config", this.instanceDir()), 0755)
	swallow(err)
	chown(fmt.Sprintf("%s/config", this.instanceDir()), this.userInfo)

	type templateVar struct {
		KafkaBase             string
		BrokerId              string
		TcpPort               string
		Ip                    string
		User                  string
		ZkChroot              string
		ZkAddrs               string
		InstanceDir           string
		LogDirs               string
		IoThreads             string
		NetworkThreads        string
		InfluxReporterEnabled string
		InfluxDbHost          string
		InfluxDbPort          string
	}
	if this.influxDbAddr != "" {
		this.influxDbHost, this.influxdbPort, err = net.SplitHostPort(this.influxDbAddr)
		if err != nil {
			this.Ui.Error(err.Error())
			return 2
		}
		if this.influxDbHost == "" || this.influxdbPort == "" {
			this.Ui.Error("empty influxdb host or port")
			return 2
		}
	}

	influxReporterEnabled := "false"
	if this.influxDbHost != "" {
		influxReporterEnabled = "true"
	}
	data := templateVar{
		ZkAddrs:               this.zkzone.ZkAddrs(),
		ZkChroot:              zkchroot,
		KafkaBase:             this.kafkaBaseDir,
		BrokerId:              this.brokerId,
		Ip:                    this.ip,
		InstanceDir:           this.instanceDir(),
		User:                  this.runAs,
		TcpPort:               this.tcpPort,
		LogDirs:               this.logDirs,
		InfluxReporterEnabled: influxReporterEnabled,
		InfluxDbHost:          this.influxDbHost,
		InfluxDbPort:          this.influxdbPort,
	}
	data.IoThreads = strconv.Itoa(3 * len(strings.Split(data.LogDirs, ",")))
	networkThreads := ctx.NumCPU() / 2
	if networkThreads < 2 {
		networkThreads = 2
	}
	data.NetworkThreads = strconv.Itoa(networkThreads) // TODO not used yet

	// create the log.dirs directory and chown to sre
	logDirs := strings.Split(this.logDirs, ",")
	for _, logDir := range logDirs {
		if this.dryRun {
			this.Ui.Output(fmt.Sprintf("mkdir %s and chown to %s",
				logDir, this.runAs))
		}

		swallow(os.MkdirAll(logDir, 0755))
		chown(logDir, this.userInfo)
	}

	// package the kafka runtime together
	if !gio.DirExists(this.kafkaLibDir()) {
		this.installKafka()
	}

	// bin
	writeFileFromTemplate("template/bin/kafka-topics.sh",
		fmt.Sprintf("%s/bin/kafka-topics.sh", this.instanceDir()), 0755, nil, this.userInfo)
	writeFileFromTemplate("template/bin/kafka-reassign-partitions.sh",
		fmt.Sprintf("%s/bin/kafka-reassign-partitions.sh", this.instanceDir()), 0755, nil, this.userInfo)
	writeFileFromTemplate("template/bin/kafka-preferred-replica-election.sh",
		fmt.Sprintf("%s/bin/kafka-preferred-replica-election.sh", this.instanceDir()), 0755, nil, this.userInfo)

	writeFileFromTemplate("template/bin/kafka-run-class.sh",
		fmt.Sprintf("%s/bin/kafka-run-class.sh", this.instanceDir()), 0755, data, this.userInfo)
	writeFileFromTemplate("template/bin/kafka-server-start.sh",
		fmt.Sprintf("%s/bin/kafka-server-start.sh", this.instanceDir()), 0755, data, this.userInfo)
	writeFileFromTemplate("template/bin/setenv.sh",
		fmt.Sprintf("%s/bin/setenv.sh", this.instanceDir()), 0755, data, this.userInfo)

	// /etc/init.d/
	writeFileFromTemplate("template/init.d/kafka",
		fmt.Sprintf("/etc/init.d/%s", this.clusterName()), 0755, data, nil)

	// config
	writeFileFromTemplate("template/config/server.properties",
		fmt.Sprintf("%s/config/server.properties", this.instanceDir()), 0644, data, this.userInfo)
	writeFileFromTemplate("template/config/log4j.properties",
		fmt.Sprintf("%s/config/log4j.properties", this.instanceDir()), 0644, data, this.userInfo)

	this.Ui.Warn(fmt.Sprintf("NOW, please run the following command:"))
	this.Ui.Output(color.Red("confirm log.retention.hours"))
	this.Ui.Output(color.Red("chkconfig --add %s", this.clusterName()))
	this.Ui.Output(color.Red("/etc/init.d/%s start", this.clusterName()))

	return
}
예제 #23
0
파일: clusters.go 프로젝트: funkygao/gafka
func (this *Clusters) printClusters(zkzone *zk.ZkZone, clusterPattern string, port string) {
	if this.registeredBrokers {
		this.printRegisteredBrokers(zkzone)
		return
	}

	type clusterInfo struct {
		name, path         string
		nickname           string
		topicN, partitionN int
		err                string
		priority           int
		public             bool
		retention          int
		replicas           int
		brokerInfos        []zk.BrokerInfo
	}
	clusters := make([]clusterInfo, 0)
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		ci := clusterInfo{
			name: zkcluster.Name(),
			path: zkcluster.Chroot(),
		}
		if this.neat {
			clusters = append(clusters, ci)
			return
		}

		// verbose mode, will calculate topics and partition count
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			ci.err = "no live brokers"
			clusters = append(clusters, ci)
			return
		}

		if port != "" {
			for _, hostport := range brokerList {
				_, p, err := net.SplitHostPort(hostport)
				swallow(err)

				if p != port {
					return
				}
			}
		}

		info := zkcluster.RegisteredInfo()
		if this.publicOnly && !info.Public {
			return
		}

		if !this.verbose {
			ci.brokerInfos = info.Roster
			clusters = append(clusters, ci)
			return
		}

		kfk, err := sarama.NewClient(brokerList, saramaConfig())
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		topics, err := kfk.Topics()
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		partitionN := 0
		for _, topic := range topics {
			partitions, err := kfk.Partitions(topic)
			if err != nil {
				ci.err = err.Error()
				clusters = append(clusters, ci)
				continue
			}

			partitionN += len(partitions)
		}

		clusters = append(clusters, clusterInfo{
			name:        zkcluster.Name(),
			nickname:    info.Nickname,
			path:        zkcluster.Chroot(),
			topicN:      len(topics),
			partitionN:  partitionN,
			retention:   info.Retention,
			public:      info.Public,
			replicas:    info.Replicas,
			priority:    info.Priority,
			brokerInfos: info.Roster,
		})
	})

	this.Ui.Output(fmt.Sprintf("%s: %d", zkzone.Name(), len(clusters)))
	if this.verbose {
		// 2 loop: 1. print the err clusters 2. print the good clusters
		for _, c := range clusters {
			if c.err == "" {
				continue
			}

			this.Ui.Output(fmt.Sprintf("%30s: %s %s", c.name, c.path,
				color.Red(c.err)))
		}

		// loop2
		for _, c := range clusters {
			if c.err != "" {
				continue
			}

			this.Ui.Output(fmt.Sprintf("%30s: %s",
				c.name, c.path))
			brokers := []string{}
			for _, broker := range c.brokerInfos {
				if this.ipInNumber {
					brokers = append(brokers, fmt.Sprintf("%d/%s:%d", broker.Id, broker.Host, broker.Port))
				} else {
					brokers = append(brokers, fmt.Sprintf("%d/%s", broker.Id, broker.NamedAddr()))
				}
			}
			if len(brokers) > 0 {
				sort.Strings(brokers)
				this.Ui.Info(color.Green("%31s %s", " ", strings.Join(brokers, ", ")))
			}

			this.Ui.Output(strings.Repeat(" ", 4) +
				color.Green("nick:%s public:%v topics:%d partitions:%d replicas:%d retention:%dh",
					c.nickname, c.public,
					c.topicN, c.partitionN, c.replicas, c.retention))
		}

		return
	}

	// not verbose mode
	hostsWithoutDnsRecords := make([]string, 0)
	for _, c := range clusters {
		this.Ui.Output(fmt.Sprintf("%30s: %s", c.name, c.path))
		brokers := []string{}
		for _, broker := range c.brokerInfos {
			if this.ipInNumber {
				brokers = append(brokers, fmt.Sprintf("%d/%s:%d", broker.Id, broker.Host, broker.Port))
			} else {
				brokers = append(brokers, fmt.Sprintf("%d/%s", broker.Id, broker.NamedAddr()))
			}

			if broker.Addr() == broker.NamedAddr() {
				hostsWithoutDnsRecords = append(hostsWithoutDnsRecords, fmt.Sprintf("%s:%s", c.name, broker.Addr()))
			}
		}
		if len(brokers) > 0 {
			sort.Strings(brokers)
			this.Ui.Info(color.Green("%31s %s", " ", strings.Join(brokers, ", ")))
		} else {
			this.Ui.Warn(fmt.Sprintf("%31s no live registered brokers", " "))
		}
	}

	if len(hostsWithoutDnsRecords) > 0 {
		this.Ui.Warn("brokers without dns record:")
		for _, broker := range hostsWithoutDnsRecords {
			parts := strings.SplitN(broker, ":", 2)
			this.Ui.Output(fmt.Sprintf("%30s: %s", parts[0], color.Yellow(parts[1])))
		}
	}

}
예제 #24
0
파일: topics.go 프로젝트: funkygao/gafka
func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) {
	echoBuffer := func(lines []string) {
		for _, l := range lines {
			this.Ui.Output(l)
		}
	}

	linesInTopicMode := make([]string, 0)
	if this.verbose {
		linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode)
	}

	// get all alive brokers within this cluster
	brokers := zkcluster.Brokers()
	if len(brokers) == 0 {
		linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ",
			color.Red("%s empty brokers", zkcluster.Name())), linesInTopicMode)
		echoBuffer(linesInTopicMode)
		return
	}

	if this.verbose {
		sortedBrokerIds := make([]string, 0, len(brokers))
		for brokerId, _ := range brokers {
			sortedBrokerIds = append(sortedBrokerIds, brokerId)
		}
		sort.Strings(sortedBrokerIds)
		for _, brokerId := range sortedBrokerIds {
			if this.ipInNumber {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
					color.Green(brokerId), brokers[brokerId]), linesInTopicMode)
			} else {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
					color.Green(brokerId), brokers[brokerId].NamedString()), linesInTopicMode)
			}

		}
	}

	kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig())
	if err != nil {
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%+v %s", " ",
				zkcluster.BrokerList(), err.Error()), linesInTopicMode)
		}

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		if this.topicPattern == "" && this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ",
				color.Magenta("no topics")), linesInTopicMode)
			echoBuffer(linesInTopicMode)
		}

		return
	}

	sortedTopics := make([]string, 0, len(topics))
	for _, t := range topics {
		sortedTopics = append(sortedTopics, t)
	}
	sort.Strings(sortedTopics)

	topicsCtime := zkcluster.TopicsCtime()
	hasTopicMatched := false
	for _, topic := range sortedTopics {
		if !patternMatched(topic, this.topicPattern) {
			continue
		}

		if this.since > 0 && time.Since(topicsCtime[topic]) > this.since {
			continue
		}

		this.topicN++

		hasTopicMatched = true
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Cyan(topic), linesInTopicMode)
		}

		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		swallow(err)
		partions, err := kfk.Partitions(topic)
		swallow(err)
		if len(alivePartitions) != len(partions) {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %s P: %s/%+v",
				zkcluster.Name(), color.Cyan("%-50s", topic), color.Red("partial dead"), color.Green("%+v", alivePartitions), partions), linesInTopicMode)
		}

		replicas, err := kfk.Replicas(topic, partions[0])
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partions[0], err))
		}

		this.partitionN += len(partions)
		if !this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR %s",
				zkcluster.Name(),
				color.Cyan("%-50s", topic),
				len(partions), len(replicas),
				gofmt.PrettySince(topicsCtime[topic])), linesInTopicMode)
			continue
		}

		for _, partitionID := range alivePartitions {
			leader, err := kfk.Leader(topic, partitionID)
			swallow(err)

			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partitionID, err))
			}

			isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)
			isrMtimeSince := gofmt.PrettySince(isrMtime)
			if time.Since(isrMtime).Hours() < 24 {
				// ever out of sync last 24h
				isrMtimeSince = color.Magenta(isrMtimeSince)
			}

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			latestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetNewest)
			swallow(err)

			oldestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetOldest)
			swallow(err)

			if this.count > 0 && (latestOffset-oldestOffset) < this.count {
				continue
			}

			this.totalMsgs += latestOffset - oldestOffset
			this.totalOffsets += latestOffset
			if !underReplicated {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%+v Offset:%16s - %-16s Num:%-15s %s-%s",
					partitionID,
					color.Green("%d", leader.ID()), replicas, isr,
					gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset),
					gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode)
			} else {
				// use red for alert
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%s Offset:%16s - %-16s Num:%-15s %s-%s",
					partitionID,
					color.Green("%d", leader.ID()), replicas, color.Red("%+v", isr),
					gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset),
					gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode)
			}
		}
	}

	if this.topicPattern != "" {
		if hasTopicMatched {
			echoBuffer(linesInTopicMode)
		}

	} else {
		echoBuffer(linesInTopicMode)
	}
}
예제 #25
0
func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) []string {
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Warn(fmt.Sprintf("%s empty brokers", zkcluster.Name()))
		return nil
	}

	kfk, err := sarama.NewClient(brokerList, saramaConfig())
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %+v %s", zkcluster.Name(), brokerList, err.Error()))

		return nil
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return nil
	}

	lines := make([]string, 0, 10)
	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch writable partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] has %s partitions: %+v/%+v", zkcluster.Name(),
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("%s topic[%s] P:%d: %v", zkcluster.Name(), topic, partitionID, err))
				continue
			}

			isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				lines = append(lines, fmt.Sprintf("\t%s Partition:%d/%s Leader:%d Replicas:%+v Isr:%+v/%s Offset:%d-%d Num:%d",
					topic, partitionID,
					gofmt.PrettySince(partitionCtime),
					leader.ID(), replicas, isr,
					gofmt.PrettySince(isrMtime),
					oldestOffset, latestOffset, latestOffset-oldestOffset))
			}
		}
	}

	return lines
}
예제 #26
0
파일: mysql.go 프로젝트: xinghuwang/toolbox
func main() {
	var wg sync.WaitGroup
	t0 := time.Now()
	idgen = make(chan int, 1000)
	go func() {
		id := 0
		for {
			id++
			idgen <- id
		}
	}()

	var rows int
	switch opts.op {
	case "insert":
		for i := 0; i < opts.concurrent; i++ {
			wg.Add(1)
			go func(seq int) {
				insert(seq, opts.loop)
				wg.Done()
			}(i)
		}
		rows = opts.loop * opts.concurrent

	case "update":
		for i := 0; i < opts.concurrent; i++ {
			wg.Add(1)
			go func(seq int) {
				update(seq, opts.loop)
				wg.Done()
			}(i)
		}
		rows = opts.loop * opts.concurrent

	case "delete":
		db, err := sql.Open("mysql", dsn)
		dieIfErr(err)
		rs, err := db.Exec("DELETE FROM user")
		if err != nil {
			fmt.Println(err)
		} else {
			if n, err := rs.RowsAffected(); err != nil {
				fmt.Println(err)
			} else {
				fmt.Printf("%d rows deleted\n", n)
				rows = int(n)
			}
		}
		db.Close()

	default:
		fmt.Println("unknown operation:", opts.op)
	}

	wg.Wait()
	elapsed := time.Since(t0)
	dur := int(elapsed.Seconds())
	if dur == 0 {
		dur = 1
	}
	fmt.Printf("%s elapsed:%s, rows:%d %s:%d\n", opts.op, elapsed, rows,
		color.Red("qps"),
		rows/dur)

}