func main() {
// Initialise our configuration from flags
var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
var startIndex = flag.Int64("index", 1, "Index to start reading messages from")
var quantity = flag.Int("quantity", 1, "Target number of messages to retrieve. A larger or smaller number may be returned")
var waitForMessages = flag.Bool("wait", false, "Wait for messages to be available")
var enableDebug = flag.Bool("debug", false, "Enable debug logging")
var tail = flag.Bool("tail", false, "Keeps reading all available messages, implies -wait=true")
flag.Parse()
if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
flag.PrintDefaults()
return
}
if *fbConnect == REQUIRED {
Print("forestbus connection string missing.\n")
flag.PrintDefaults()
return
}
clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)
if len(peers) == 0 {
Print("At least one node name is required.\n")
}
if topic == "" {
fmt.Printf("Topic is missing.\n")
flag.PrintDefaults()
return
}
var client *forestbus.Client
if *enableDebug {
client = forestbus.NewClient(clusterID, peers, forestbus.ClientEnableDebug())
} else {
client = forestbus.NewClient(clusterID, peers)
}
maxMsgID, err := client.GetTopicMaxAvailableIndex(topic)
if err != nil {
Print("Error determining maximum available message ID: %v\n", err)
return
}
if maxMsgID == 0 {
Print("Commit index is zero due to a full cluster restart.)\n")
return
}
if *tail {
*waitForMessages = true
}
looping := true
recievedCount := 0
firstIndex := *startIndex
for looping {
msgs, nextID, err := client.GetMessages(topic, *startIndex, *quantity, *waitForMessages)
if err != nil {
Print("Error getting messages: %v\n", err)
return
}
if len(msgs) == 0 {
if nextID == 0 {
Print("Commit index is zero due to a full cluster restart.)\n")
return
} else if nextID == (*startIndex + 1) {
// Nothing came back - are we planning to wait for more?
if !*waitForMessages {
looping = false
}
} else if nextID == *startIndex {
// Index 1 not available due to clean up
firstIndex = nextID
}
}
for _, msg := range msgs {
os.Stdout.Write(msg)
os.Stdout.Write(NEWLINE)
recievedCount++
if recievedCount == *quantity && !*tail {
looping = false
break
}
}
*startIndex = nextID
}
maxMsgID, err = client.GetTopicMaxAvailableIndex(topic)
if err != nil {
Print("Error determining maximum available message ID: %v\n", err)
return
}
Print("Recieved %v messages from index %v\n", recievedCount, firstIndex)
Print("Maximum available message index: %v\n", maxMsgID)
}
func main() {
// Initialise our configuration from flags
var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
var numClients = flag.Int("clients", 1, "Number of clients to run simultaneously")
var numBatches = flag.Int("batches", 1, "Number of batches for each client to send")
var batchSize = flag.Int("batchSize", 1, "Number of messages in each batch sent")
var messageSize = flag.Int("msgSize", 20, "Size in bytes of the messages to be sent")
var waitForCommit = flag.Bool("wait", true, "Wait for commit on each batch send")
var enableDebug = flag.Bool("debug", false, "Enable debug logging")
var msgTemplate = flag.String("msg", "", "Message content to send - will be truncated to messageSize if required.")
flag.Usage = PrintUsage
flag.Parse()
if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
flag.PrintDefaults()
return
}
if *fbConnect == REQUIRED {
fmt.Printf("forestbus connection string missing.\n")
flag.PrintDefaults()
return
}
clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)
if len(peers) == 0 {
PrintUsage()
fmt.Printf("\nError: At least one node name is required.\n")
return
}
if topic == "" {
PrintUsage()
fmt.Printf("\nError: -topic is missing.\n")
return
}
wg := sync.WaitGroup{}
startTime := time.Now()
for clientCount := 0; clientCount < *numClients; clientCount++ {
wg.Add(1)
var client *forestbus.Client
if *enableDebug {
client = forestbus.NewClient(clusterID, peers, forestbus.ClientEnableDebug())
} else {
client = forestbus.NewClient(clusterID, peers)
}
go clientSend(topic, &wg, client, *numBatches, *batchSize, *messageSize, *msgTemplate, *waitForCommit)
}
wg.Wait()
endTime := time.Now()
totalMessages := *numClients * *numBatches * *batchSize
totalBytes := float64(totalMessages * *messageSize)
durationSeconds := endTime.Sub(startTime).Seconds()
fmt.Printf("%v messages (%v) in %v [%v msgs/sec and %v/sec]\n", totalMessages, formatBytes(totalBytes), endTime.Sub(startTime), fmt.Sprintf("%.f", float64(totalMessages)/durationSeconds), formatBytes(float64(totalBytes)/durationSeconds))
log.Printf("Done\n")
}
func main() {
// Initialise our configuration from flags
var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
var startNumber = flag.Int("start", 1, "Number of clients / batches to start at.")
var endNumber = flag.Int("end", 1, "Number of clients / batches to end at.")
var stepSize = flag.Int("step", 1, "Step size to increase the number of clients / batches")
var numBatches = flag.Int("batches", 1, "Number of batches for each client to send")
var messageSize = flag.Int("msgSize", 20, "Size in bytes of the messages to be sent")
var waitForCommit = flag.Bool("wait", true, "Wait for commit on each batch send")
var enableDebug = flag.Bool("debug", false, "Enable debug logging")
var msgTemplate = flag.String("msg", "", "Message content to send - will be truncated to messageSize if required.")
flag.Parse()
if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
flag.PrintDefaults()
return
}
if *fbConnect == REQUIRED {
fmt.Printf("forestbus connection string missing.\n")
flag.PrintDefaults()
return
}
clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)
if len(peers) == 0 {
fmt.Printf("At least one node name is required.\n")
}
if topic == "" {
fmt.Printf("topic missing.\n")
flag.PrintDefaults()
return
}
wg := sync.WaitGroup{}
numClients := *startNumber
batchSize := *startNumber
tout, err := os.Create("perf-test-out.csv")
if err != nil {
panic("Error opening output file: " + err.Error())
}
csvout := csv.NewWriter(tout)
csvout.Write([]string{"Count", "Clients run 1", "Clients run 2", "Clients run 3", "Clients Avg", "Batch run 1", "Batch run 2", "Batch run 3", "Batch Avg", "Combined run 1", "Combined run 2", "Combined run 3", "Combined Avg"})
rowResults := make([]string, 0, 13)
// Create the clients now to avoid having to constantly re-connect.
clients := make([]*forestbus.Client, 0, *endNumber)
for i := 0; i < *endNumber; i++ {
var client *forestbus.Client
if *enableDebug {
client = forestbus.NewClient(clusterID, peers, forestbus.ClientEnableDebug())
} else {
client = forestbus.NewClient(clusterID, peers)
}
clients = append(clients, client)
}
for testnumber := *startNumber; testnumber <= *endNumber; testnumber += *stepSize {
rowResults = append(rowResults, fmt.Sprintf("%v", testnumber))
for testType := 0; testType < 3; testType++ {
switch testType {
case 0:
numClients = testnumber
batchSize = 1
case 1:
numClients = 1
batchSize = testnumber
case 2:
numClients = testnumber
batchSize = testnumber
}
runRes := make([]float64, 0, 3)
for testrun := 0; testrun < 3; testrun++ {
startTime := time.Now()
for clientCount := 0; clientCount < numClients; clientCount++ {
wg.Add(1)
go clientSend(topic, &wg, clients[clientCount], *numBatches, batchSize, *messageSize, *msgTemplate, *waitForCommit)
}
wg.Wait()
endTime := time.Now()
totalMessages := numClients * *numBatches * batchSize
totalBytes := float64(totalMessages * *messageSize)
durationSeconds := endTime.Sub(startTime).Seconds()
runRes = append(runRes, float64(totalMessages)/durationSeconds)
rowResults = append(rowResults, fmt.Sprintf("%.f", float64(totalMessages)/durationSeconds))
fmt.Printf("%v messages (%v) in %v [%v msgs/sec and %v/sec]\n", totalMessages, formatBytes(totalBytes), endTime.Sub(startTime), fmt.Sprintf("%.f", float64(totalMessages)/durationSeconds), formatBytes(float64(totalBytes)/durationSeconds))
}
avg := (runRes[0] + runRes[1] + runRes[2]) / 3
rowResults = append(rowResults, fmt.Sprintf("%.f", avg))
runRes = runRes[:0]
}
csvout.Write(rowResults)
rowResults = rowResults[:0]
}
csvout.Flush()
tout.Close()
log.Printf("Done\n")
}
func main() {
// Initialise our configuration from flags
var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
var enableDebug = flag.Bool("debug", false, "Enable debug logging")
flag.Usage = PrintUsage
flag.Parse()
if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
flag.PrintDefaults()
return
}
if *fbConnect == REQUIRED {
fmt.Printf("forestbus connection string missing.\n")
flag.PrintDefaults()
return
}
clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)
if len(peers) == 0 {
fmt.Printf("At least one node name is required.\n")
}
if topic == "" {
fmt.Printf("topic missing.\n")
flag.PrintDefaults()
return
}
client := forestbus.NewClient(clusterID, peers)
// Try up to 7 times (~14s) if there is a leadership change or network glitch
batcher := forestbus.NewMessageBatcher(client, topic, forestbus.MessageBatcherRetries(forestbus.UNLIMITED_RETRIES))
// Listen for signals
exitSignal := make(chan os.Signal, 10)
signal.Notify(exitSignal, os.Interrupt, os.Kill)
// Setup the stdin reader
reader := NewStdInReader()
go reader.ReadStdIn()
// Setup the message sender
sender := NewMsgSender(batcher)
go sender.Send(reader.MsgChan)
var state State = RUNNING
running := true
if *enableDebug {
log.Printf("Reading from stdin to send to topic %v\n", topic)
}
var sentCount, errCount int
for running {
select {
case <-exitSignal:
switch state {
case RUNNING:
// Ask the reader to stop generating new messages
fmt.Printf("Exit detected - shutting down\n")
reader.RequestStop()
batcher.Close()
case DRAINING:
// Time to abort
fmt.Printf("Abort detected - shutting down\n")
sender.RequestStop()
batcher.Close()
state = ABORTING
case COMPLETING:
fmt.Printf("Abort detected - shutting down\n")
state = ABORTING
}
case <-reader.FinishedChan:
if state == RUNNING {
state = DRAINING
}
case <-sender.FinishedChan:
// Draining or running or abort has finished - all messages are in the MessageBatcher
fmt.Printf("Messages sent - completing\n")
state = COMPLETING
batcher.Close()
// Clear any final message stuck in Client
client.Close()
// Now we are done
running = false
case result := <-sender.ResultChan:
if sendErr := result.GetError(); sendErr != nil {
fmt.Printf("Error sending message: %v\n", sendErr)
// Tell our stdin reader to stop reading and wait for it to confirm
reader.RequestStop()
errCount++
} else {
sentCount++
}
}
}
// Wait for all message results to come back
waitToClear := true
if (errCount + sentCount) == reader.Count {
waitToClear = false
}
timeout := time.NewTimer(2 * time.Second)
for waitToClear {
select {
case result := <-sender.ResultChan:
if sendErr := result.GetError(); sendErr != nil {
errCount++
} else {
sentCount++
}
if (errCount + sentCount) == reader.Count {
waitToClear = false
}
case <-timeout.C:
waitToClear = false
}
}
fmt.Printf("Read %v messages, sent %v messages succesfully and %v errors.\n", reader.Count, sentCount, errCount)
}