func main() {
mf := SpawnMessageFetcher(pixyAddr, topic, group)
progressCh := make(chan progress)
go func() {
var wg sync.WaitGroup
chunkSize := count / threads
for i := 0; i < threads; i++ {
wg.Add(1)
go func() {
defer wg.Done()
var recentProgress progress
checkpoint := time.Now()
for j := 1; j < chunkSize; j++ {
select {
case msg, ok := <-mf.Messages():
if !ok {
goto done
}
recentProgress.count += 1
recentProgress.bytes += int64(len(msg))
now := time.Now()
if now.Sub(checkpoint) > reportingPeriod {
checkpoint = now
progressCh <- recentProgress
recentProgress.count, recentProgress.bytes = 0, 0
}
case <-time.After(longPollingTimeout):
if waitForMore {
continue
}
goto done
}
}
done:
progressCh <- recentProgress
}()
}
wg.Wait()
go func() {
mf.Stop()
}()
// Read the remaining fetched but not processed messages.
var lastProgress progress
for msg := range mf.Messages() {
lastProgress.bytes = int64(len(msg))
lastProgress.count += 1
}
progressCh <- lastProgress
close(progressCh)
}()
begin := time.Now()
checkpoint := begin
var totalProgress progress
for progress := range progressCh {
totalProgress.count += progress.count
totalProgress.bytes += progress.bytes
now := time.Now()
totalTook := now.Sub(begin)
took := now.Sub(checkpoint)
checkpoint = now
tookSec := float64(took) / float64(time.Second)
fmt.Printf("Consuming... %d(%s) for %s at %dmsg(%s)/sec\n",
totalProgress.count, prettyfmt.Bytes(totalProgress.bytes), totalTook,
int64(float64(progress.count)/tookSec),
prettyfmt.Bytes(int64(float64(progress.bytes)/tookSec)))
}
took := time.Now().Sub(begin)
tookSec := float64(took) / float64(time.Second)
fmt.Printf("Consumed %d(%s) for %s at %dmsg(%s)/sec\n",
totalProgress.count, prettyfmt.Bytes(totalProgress.bytes), took,
int64(float64(totalProgress.count)/tookSec),
prettyfmt.Bytes(int64(float64(totalProgress.bytes)/tookSec)))
}
func main() {
msg := genmessage(size)
progressCh := make(chan int)
useUnixDomainSocket := false
var baseURL string
if strings.HasPrefix(pixyAddr, "/") {
fmt.Printf("Using UDS client for %s\n", pixyAddr)
baseURL = "http://_"
useUnixDomainSocket = true
} else {
fmt.Printf("Using net client for %s\n", pixyAddr)
baseURL = fmt.Sprintf("http://%s", pixyAddr)
}
go func() {
var wg sync.WaitGroup
chunkSize := count / threads
for i := 0; i < threads; i++ {
messageIndexBegin := chunkSize * i
messageIndexEnd := messageIndexBegin + chunkSize
if count-messageIndexEnd < chunkSize {
messageIndexEnd = count
}
wg.Add(1)
go func() {
defer wg.Done()
var clt http.Client
if useUnixDomainSocket {
dial := func(proto, addr string) (net.Conn, error) {
return net.Dial("unix", pixyAddr)
}
clt.Transport = &http.Transport{Dial: dial}
}
recentProgress := 0
checkpoint := time.Now()
for i := messageIndexBegin; i < messageIndexEnd; i++ {
var URL string
if isSync {
URL = fmt.Sprintf("%s/topics/%s/messages?sync&key=%d", baseURL, topic, i)
} else {
URL = fmt.Sprintf("%s/topics/%s/messages?key=%d", baseURL, topic, i)
}
res, err := clt.Post(URL, "text/plain", bytes.NewReader(msg))
if err != nil {
panic(fmt.Errorf("failed to POST %s, err=(%s)", URL, err))
}
if res.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(res.Body)
if err != nil {
panic(err)
}
panic(fmt.Sprintf("%d %s", res.StatusCode, body))
}
res.Body.Close()
recentProgress += 1
if time.Now().Sub(checkpoint) > reportingPeriod {
progressCh <- recentProgress
recentProgress = 0
}
}
progressCh <- recentProgress
}()
}
wg.Wait()
close(progressCh)
}()
fmt.Printf("Producing...")
begin := time.Now()
totalProgress := 0
for progress := range progressCh {
totalProgress += progress
took := time.Now().Sub(begin)
tookSec := float64(took) / float64(time.Second)
fmt.Printf("\rProducing... %d/%d for %s at %dmsg(%s)/sec",
totalProgress, count, took, int64(float64(totalProgress)/tookSec),
prettyfmt.Bytes(int64(float64(size*totalProgress)/tookSec)))
}
took := time.Now().Sub(begin)
tookSec := float64(took) / float64(time.Second)
fmt.Printf("\rProduced %d messages of size %d for %s at %dmsg(%s)/sec\n",
totalProgress, size, took, int64(float64(totalProgress)/tookSec),
prettyfmt.Bytes(int64(float64(size*totalProgress)/tookSec)))
}