Beispiel #1
0
func cmdBenchmarkSet() error {

	if len(globalBrokerList) == 0 {
		return errors.NotValidf("broker list")
	}
	if len(globalTopic) == 0 {
		return errors.NotValidf("Topic")
	}

	sendString := utils.GenTestMessage(globalMsgLength)
	producerConfig := siesta_producer.NewProducerConfig()
	producerConfig.Linger = time.Millisecond
	connConfig := siesta.NewConnectorConfig()
	brokerList := strings.Split(globalBrokerList, ",")
	producerConfig.BrokerList = brokerList
	connConfig.BrokerList = brokerList

	log.Printf("%v", brokerList)
	connector, err := siesta.NewDefaultConnector(connConfig)
	if err != nil {
		return errors.Trace(err)
	}
	//	go func() {
	//		timeout := time.Tick(producerConfig.MetadataExpire / 2)
	//		for {
	//			<-timeout
	//			connector.RefreshMetadata([]string{globalTopic})
	//		}
	//	}()

	producer := siesta_producer.NewKafkaProducer(producerConfig,
		siesta_producer.ByteSerializer,
		siesta_producer.ByteSerializer,
		connector)

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration,
		func(bt *utils.BenchmarkTester, index int) error {

			record := &siesta_producer.ProducerRecord{
				Topic: globalTopic,
				Value: []byte(sendString),
			}

			recordMetadata := <-producer.Send(record)
			if recordMetadata.Error == siesta.ErrNoError {
				return nil
			}
			return recordMetadata.Error
		}, nil)
	return errors.Trace(bt.Run())
}
Beispiel #2
0
func cmdBenchmarkGet() error {

	if len(globalZkList) == 0 {
		return errors.NotValidf("zookeeper list")
	}

	if len(globalTopic) == 0 {
		return errors.NotValidf("Topic")
	}

	message := make(chan *go_kafka_client.Message)
	consumConfig := go_kafka_client.DefaultConsumerConfig()
	consumConfig.Groupid = "BenchmarkGroup-Test"
	consumConfig.AutoOffsetReset = go_kafka_client.SmallestOffset
	zkConfig := go_kafka_client.NewZookeeperConfig()
	zkConfig.ZookeeperConnect = strings.Split(globalZkList, ",")
	consumConfig.Coordinator = go_kafka_client.NewZookeeperCoordinator(zkConfig)

	consumConfig.Strategy =
		func(_ *go_kafka_client.Worker, msg *go_kafka_client.Message,
			id go_kafka_client.TaskId) go_kafka_client.WorkerResult {

			message <- msg
			return go_kafka_client.NewSuccessfulResult(id)
		}

	consumConfig.WorkerFailureCallback =
		func(_ *go_kafka_client.WorkerManager) go_kafka_client.FailedDecision {
			return go_kafka_client.CommitOffsetAndContinue
		}

	consumConfig.WorkerFailedAttemptCallback =
		func(_ *go_kafka_client.Task, _ go_kafka_client.WorkerResult) go_kafka_client.FailedDecision {
			return go_kafka_client.CommitOffsetAndContinue
		}

	consumer := go_kafka_client.NewConsumer(consumConfig)
	topicCountMap := make(map[string]int)
	topicCountMap[globalTopic] = 1
	go consumer.StartStatic(topicCountMap)

	time.Sleep(2e9)
	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration, func(bt *utils.BenchmarkTester, index int) error {

		<-message
		return nil
	}, nil)

	return errors.Trace(bt.Run())
}
Beispiel #3
0
func cmdBenchmarkSetNoAck() error {

	if len(globalBrokerList) == 0 {
		return errors.NotValidf("broker list")
	}
	if len(globalTopic) == 0 {
		return errors.NotValidf("Topic")
	}

	sendString := utils.GenTestMessage(globalMsgLength)
	producerConfig := siesta_producer.NewProducerConfig()
	producerConfig.ClientID = "Benchmark"
	producerConfig.RequiredAcks = 0
	connConfig := siesta.NewConnectorConfig()
	brokerList := strings.Split(globalBrokerList, ",")
	producerConfig.BrokerList = brokerList
	connConfig.BrokerList = brokerList

	log.Printf("%v", brokerList)
	connector, err := siesta.NewDefaultConnector(connConfig)
	if err != nil {
		return errors.Trace(err)
	}

	producer := siesta_producer.NewKafkaProducer(producerConfig,
		siesta_producer.ByteSerializer,
		siesta_producer.ByteSerializer,
		connector)

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration,
		func(bt *utils.BenchmarkTester, index int) error {

			record := &siesta_producer.ProducerRecord{
				Topic: globalTopic,
				Value: []byte(sendString),
			}

			recordMetadata := <-producer.Send(record)
			if recordMetadata.Error == siesta.ErrNoError {
				return nil
			}
			return recordMetadata.Error
		}, nil)
	return errors.Trace(bt.Run())
}
Beispiel #4
0
func benchmarkMCGet() error {

	key := fmt.Sprintf("%s.%s", globalQueue, globalBiz)
	log.Printf("Test Key: %s", key)

	mc := memcache.New(globalHost)

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration, func(bt *utils.BenchmarkTester, index int) error {

		_, err := mc.Get(key)
		if err != nil {
			return err
		}

		return nil
	}, nil)
	return errors.Trace(bt.Run())
}
Beispiel #5
0
func benchmarkMCSet() error {

	key := fmt.Sprintf("%s.%s", globalQueue, globalBiz)
	sendString := utils.GenTestMessage(globalMsgLength)
	log.Printf("Test Key: %s, Data: %s", key, sendString)

	mc := memcache.New(globalHost)

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration, func(bt *utils.BenchmarkTester, index int) error {

		err := mc.Set(&memcache.Item{Key: key, Value: []byte(sendString)})
		if err != nil {
			return err
		}

		return nil
	}, nil)
	return errors.Trace(bt.Run())
}
Beispiel #6
0
func benchmarkHttpGet() error {

	url := fmt.Sprintf("http://%s/msg?action=receive&queue=%s&group=%s",
		globalHost, globalQueue, globalBiz)
	log.Printf("Test URL: %s", url)

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration, func(bt *utils.BenchmarkTester, index int) error {

		resp, err := http.Get(url)
		if err != nil {
			return err
		}
		if resp.StatusCode != 200 {
			return fmt.Errorf("http code %d", resp.StatusCode)
		}

		return nil
	}, nil)
	return errors.Trace(bt.Run())
}
Beispiel #7
0
func benchmarkHttpSet() error {

	url := fmt.Sprintf("http://%s/msg", globalHost)
	sendString := fmt.Sprintf("action=send&queue=%s&group=%s&msg=%s",
		globalQueue, globalBiz, utils.GenTestMessage(globalMsgLength))
	log.Printf("Test URL: %s, Data: %s", url, sendString)

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration, func(bt *utils.BenchmarkTester, index int) error {

		body := strings.NewReader(sendString)
		resp, err := http.Post(url, "application/x-www-form-urlencoded", body)
		if err != nil {
			return err
		}
		if resp.StatusCode != 200 {
			return fmt.Errorf("http code %d", resp.StatusCode)
		}
		return nil
	}, nil)
	return errors.Trace(bt.Run())
}
Beispiel #8
0
func benchmarkHttpLatency() error {

	getURL := fmt.Sprintf("http://%s/msg?action=receive&queue=%s&group=%s",
		globalHost, globalQueue, globalBiz)
	setURL := fmt.Sprintf("http://%s/msg", globalHost)

	result := new([6]int64)
	statis := func(s int64) {
		switch {
		case s < 2:
			atomic.AddInt64(&(result[0]), 1)
		case s >= 2 && s < 5:
			atomic.AddInt64(&(result[1]), 1)
		case s >= 5 && s < 10:
			atomic.AddInt64(&(result[2]), 1)
		case s >= 10 && s < 20:
			atomic.AddInt64(&(result[3]), 1)
		case s >= 20 && s < 50:
			atomic.AddInt64(&(result[4]), 1)
		default:
			atomic.AddInt64(&(result[5]), 1)
		}
	}

	clean := func(bt *utils.BenchmarkTester) {
		total := int64(0)
		for _, i := range result {
			total += i
		}
		fmt.Printf("Benchmark Latency Result: %d\n", total)
		fmt.Printf("\t%2d%% ops < 2ms\n", result[0]*100/total)
		fmt.Printf("\t%2d%% ops 2-5ms\n", result[1]*100/total)
		fmt.Printf("\t%2d%% ops 5-10ms\n", result[2]*100/total)
		fmt.Printf("\t%2d%% ops 10-20ms\n", result[3]*100/total)
		fmt.Printf("\t%2d%% ops 20-50ms\n", result[4]*100/total)
		fmt.Printf("\t%2d%% ops >50ms\n", result[5]*100/total)
		fmt.Printf("\n\n")
	}

	bt := utils.NewBenchmarkTester(globalConcurrentLevel, globalDuration, func(bt *utils.BenchmarkTester, index int) error {

		if index%2 == 0 {
			//producer
			body := &bytes.Buffer{}
			fmt.Fprintf(body, "action=send&queue=%s&group=%s&msg=%d",
				globalQueue, globalBiz, time.Now().UnixNano())
			resp, err := http.Post(setURL, "application/x-www-form-urlencoded", body)
			if err != nil {
				return err
			}
			if resp.StatusCode != 200 {
				return fmt.Errorf("http code %d", resp.StatusCode)
			}
			return nil
		}

		resp, err := http.Get(getURL)
		if err != nil {
			return err
		}
		if resp.StatusCode != 200 {
			return fmt.Errorf("http code %d", resp.StatusCode)
		}
		recvTime := time.Now().UnixNano()
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("resp read :%s", err)
		}
		msg := &message{}
		err = json.Unmarshal(data, msg)
		if err != nil {
			return err
		}
		sendTime, err := strconv.ParseInt(msg.Msg, 10, 64)
		if err != nil {
			return err
		}
		diff := recvTime - sendTime
		statis(diff / 1000000)
		return nil
	}, clean)
	return errors.Trace(bt.Run())
}