Esempio n. 1
0
func main() {
	mf := SpawnMessageFetcher(pixyAddr, topic, group)
	progressCh := make(chan progress)
	go func() {
		var wg sync.WaitGroup
		chunkSize := count / threads
		for i := 0; i < threads; i++ {
			wg.Add(1)
			go func() {
				defer wg.Done()

				var recentProgress progress
				checkpoint := time.Now()
				for j := 1; j < chunkSize; j++ {
					select {
					case msg, ok := <-mf.Messages():
						if !ok {
							goto done
						}

						recentProgress.count += 1
						recentProgress.bytes += int64(len(msg))
						now := time.Now()
						if now.Sub(checkpoint) > reportingPeriod {
							checkpoint = now
							progressCh <- recentProgress
							recentProgress.count, recentProgress.bytes = 0, 0
						}
					case <-time.After(longPollingTimeout):
						if waitForMore {
							continue
						}
						goto done
					}
				}
			done:
				progressCh <- recentProgress
			}()
		}
		wg.Wait()

		go func() {
			mf.Stop()
		}()
		// Read the remaining fetched but not processed messages.
		var lastProgress progress
		for msg := range mf.Messages() {
			lastProgress.bytes = int64(len(msg))
			lastProgress.count += 1
		}
		progressCh <- lastProgress
		close(progressCh)
	}()

	begin := time.Now()
	checkpoint := begin
	var totalProgress progress
	for progress := range progressCh {
		totalProgress.count += progress.count
		totalProgress.bytes += progress.bytes
		now := time.Now()
		totalTook := now.Sub(begin)
		took := now.Sub(checkpoint)
		checkpoint = now
		tookSec := float64(took) / float64(time.Second)
		fmt.Printf("Consuming... %d(%s) for %s at %dmsg(%s)/sec\n",
			totalProgress.count, prettyfmt.Bytes(totalProgress.bytes), totalTook,
			int64(float64(progress.count)/tookSec),
			prettyfmt.Bytes(int64(float64(progress.bytes)/tookSec)))
	}
	took := time.Now().Sub(begin)
	tookSec := float64(took) / float64(time.Second)
	fmt.Printf("Consumed %d(%s) for %s at %dmsg(%s)/sec\n",
		totalProgress.count, prettyfmt.Bytes(totalProgress.bytes), took,
		int64(float64(totalProgress.count)/tookSec),
		prettyfmt.Bytes(int64(float64(totalProgress.bytes)/tookSec)))
}
Esempio n. 2
0
func main() {
	msg := genmessage(size)
	progressCh := make(chan int)

	useUnixDomainSocket := false
	var baseURL string
	if strings.HasPrefix(pixyAddr, "/") {
		fmt.Printf("Using UDS client for %s\n", pixyAddr)
		baseURL = "http://_"
		useUnixDomainSocket = true
	} else {
		fmt.Printf("Using net client for %s\n", pixyAddr)
		baseURL = fmt.Sprintf("http://%s", pixyAddr)
	}

	go func() {
		var wg sync.WaitGroup
		chunkSize := count / threads
		for i := 0; i < threads; i++ {
			messageIndexBegin := chunkSize * i
			messageIndexEnd := messageIndexBegin + chunkSize
			if count-messageIndexEnd < chunkSize {
				messageIndexEnd = count
			}
			wg.Add(1)
			go func() {
				defer wg.Done()

				var clt http.Client
				if useUnixDomainSocket {
					dial := func(proto, addr string) (net.Conn, error) {
						return net.Dial("unix", pixyAddr)
					}
					clt.Transport = &http.Transport{Dial: dial}
				}

				recentProgress := 0
				checkpoint := time.Now()
				for i := messageIndexBegin; i < messageIndexEnd; i++ {
					var URL string
					if isSync {
						URL = fmt.Sprintf("%s/topics/%s/messages?sync&key=%d", baseURL, topic, i)
					} else {
						URL = fmt.Sprintf("%s/topics/%s/messages?key=%d", baseURL, topic, i)
					}
					res, err := clt.Post(URL, "text/plain", bytes.NewReader(msg))
					if err != nil {
						panic(fmt.Errorf("failed to POST %s, err=(%s)", URL, err))
					}
					if res.StatusCode != http.StatusOK {
						body, err := ioutil.ReadAll(res.Body)
						if err != nil {
							panic(err)
						}
						panic(fmt.Sprintf("%d %s", res.StatusCode, body))
					}
					res.Body.Close()
					recentProgress += 1
					if time.Now().Sub(checkpoint) > reportingPeriod {
						progressCh <- recentProgress
						recentProgress = 0
					}
				}
				progressCh <- recentProgress
			}()
		}
		wg.Wait()
		close(progressCh)
	}()

	fmt.Printf("Producing...")
	begin := time.Now()
	totalProgress := 0
	for progress := range progressCh {
		totalProgress += progress
		took := time.Now().Sub(begin)
		tookSec := float64(took) / float64(time.Second)
		fmt.Printf("\rProducing... %d/%d for %s at %dmsg(%s)/sec",
			totalProgress, count, took, int64(float64(totalProgress)/tookSec),
			prettyfmt.Bytes(int64(float64(size*totalProgress)/tookSec)))
	}
	took := time.Now().Sub(begin)
	tookSec := float64(took) / float64(time.Second)
	fmt.Printf("\rProduced %d messages of size %d for %s at %dmsg(%s)/sec\n",
		totalProgress, size, took, int64(float64(totalProgress)/tookSec),
		prettyfmt.Bytes(int64(float64(size*totalProgress)/tookSec)))
}