Esempio n. 1
0
func main() {
	// Initialise our configuration from flags

	var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
	var startIndex = flag.Int64("index", 1, "Index to start reading messages from")
	var quantity = flag.Int("quantity", 1, "Target number of messages to retrieve. A larger or smaller number may be returned")
	var waitForMessages = flag.Bool("wait", false, "Wait for messages to be available")
	var enableDebug = flag.Bool("debug", false, "Enable debug logging")
	var tail = flag.Bool("tail", false, "Keeps reading all available messages, implies -wait=true")

	flag.Parse()

	if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
		flag.PrintDefaults()
		return
	}

	if *fbConnect == REQUIRED {
		Print("forestbus connection string missing.\n")
		flag.PrintDefaults()
		return
	}

	clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)

	if len(peers) == 0 {
		Print("At least one node name is required.\n")
	}

	if topic == "" {
		fmt.Printf("Topic is missing.\n")
		flag.PrintDefaults()
		return
	}

	var client *forestbus.Client

	if *enableDebug {
		client = forestbus.NewClient(clusterID, peers, forestbus.ClientEnableDebug())
	} else {
		client = forestbus.NewClient(clusterID, peers)
	}

	maxMsgID, err := client.GetTopicMaxAvailableIndex(topic)
	if err != nil {
		Print("Error determining maximum available message ID: %v\n", err)
		return
	}

	if maxMsgID == 0 {
		Print("Commit index is zero due to a full cluster restart.)\n")
		return
	}

	if *tail {
		*waitForMessages = true
	}

	looping := true
	recievedCount := 0
	firstIndex := *startIndex

	for looping {
		msgs, nextID, err := client.GetMessages(topic, *startIndex, *quantity, *waitForMessages)

		if err != nil {
			Print("Error getting messages: %v\n", err)
			return
		}

		if len(msgs) == 0 {
			if nextID == 0 {
				Print("Commit index is zero due to a full cluster restart.)\n")
				return
			} else if nextID == (*startIndex + 1) {
				// Nothing came back - are we planning to wait for more?
				if !*waitForMessages {
					looping = false
				}
			} else if nextID == *startIndex {
				// Index 1 not available due to clean up
				firstIndex = nextID
			}
		}
		for _, msg := range msgs {
			os.Stdout.Write(msg)
			os.Stdout.Write(NEWLINE)
			recievedCount++
			if recievedCount == *quantity && !*tail {
				looping = false
				break
			}
		}
		*startIndex = nextID
	}

	maxMsgID, err = client.GetTopicMaxAvailableIndex(topic)
	if err != nil {
		Print("Error determining maximum available message ID: %v\n", err)
		return
	}

	Print("Recieved %v messages from index %v\n", recievedCount, firstIndex)
	Print("Maximum available message index: %v\n", maxMsgID)
}
Esempio n. 2
0
func main() {
	// Initialise our configuration from flags
	var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
	var numClients = flag.Int("clients", 1, "Number of clients to run simultaneously")
	var numBatches = flag.Int("batches", 1, "Number of batches for each client to send")
	var batchSize = flag.Int("batchSize", 1, "Number of messages in each batch sent")
	var messageSize = flag.Int("msgSize", 20, "Size in bytes of the messages to be sent")
	var waitForCommit = flag.Bool("wait", true, "Wait for commit on each batch send")
	var enableDebug = flag.Bool("debug", false, "Enable debug logging")
	var msgTemplate = flag.String("msg", "", "Message content to send - will be truncated to messageSize if required.")

	flag.Usage = PrintUsage

	flag.Parse()

	if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
		flag.PrintDefaults()
		return
	}

	if *fbConnect == REQUIRED {
		fmt.Printf("forestbus connection string missing.\n")
		flag.PrintDefaults()
		return
	}

	clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)

	if len(peers) == 0 {
		PrintUsage()
		fmt.Printf("\nError: At least one node name is required.\n")
		return
	}

	if topic == "" {
		PrintUsage()
		fmt.Printf("\nError: -topic is missing.\n")
		return
	}

	wg := sync.WaitGroup{}

	startTime := time.Now()
	for clientCount := 0; clientCount < *numClients; clientCount++ {
		wg.Add(1)
		var client *forestbus.Client
		if *enableDebug {
			client = forestbus.NewClient(clusterID, peers, forestbus.ClientEnableDebug())
		} else {
			client = forestbus.NewClient(clusterID, peers)
		}
		go clientSend(topic, &wg, client, *numBatches, *batchSize, *messageSize, *msgTemplate, *waitForCommit)
	}
	wg.Wait()
	endTime := time.Now()

	totalMessages := *numClients * *numBatches * *batchSize
	totalBytes := float64(totalMessages * *messageSize)
	durationSeconds := endTime.Sub(startTime).Seconds()
	fmt.Printf("%v messages (%v) in %v [%v msgs/sec and %v/sec]\n", totalMessages, formatBytes(totalBytes), endTime.Sub(startTime), fmt.Sprintf("%.f", float64(totalMessages)/durationSeconds), formatBytes(float64(totalBytes)/durationSeconds))
	log.Printf("Done\n")
}
Esempio n. 3
0
func main() {
	// Initialise our configuration from flags
	var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
	var startNumber = flag.Int("start", 1, "Number of clients / batches to start at.")
	var endNumber = flag.Int("end", 1, "Number of clients / batches to end at.")
	var stepSize = flag.Int("step", 1, "Step size to increase the number of clients / batches")
	var numBatches = flag.Int("batches", 1, "Number of batches for each client to send")
	var messageSize = flag.Int("msgSize", 20, "Size in bytes of the messages to be sent")
	var waitForCommit = flag.Bool("wait", true, "Wait for commit on each batch send")
	var enableDebug = flag.Bool("debug", false, "Enable debug logging")
	var msgTemplate = flag.String("msg", "", "Message content to send - will be truncated to messageSize if required.")

	flag.Parse()

	if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
		flag.PrintDefaults()
		return
	}

	if *fbConnect == REQUIRED {
		fmt.Printf("forestbus connection string missing.\n")
		flag.PrintDefaults()
		return
	}

	clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)

	if len(peers) == 0 {
		fmt.Printf("At least one node name is required.\n")
	}

	if topic == "" {
		fmt.Printf("topic missing.\n")
		flag.PrintDefaults()
		return
	}

	wg := sync.WaitGroup{}

	numClients := *startNumber
	batchSize := *startNumber

	tout, err := os.Create("perf-test-out.csv")
	if err != nil {
		panic("Error opening output file: " + err.Error())
	}
	csvout := csv.NewWriter(tout)
	csvout.Write([]string{"Count", "Clients run 1", "Clients run 2", "Clients run 3", "Clients Avg", "Batch run 1", "Batch run 2", "Batch run 3", "Batch Avg", "Combined run 1", "Combined run 2", "Combined run 3", "Combined Avg"})

	rowResults := make([]string, 0, 13)

	// Create the clients now to avoid having to constantly re-connect.
	clients := make([]*forestbus.Client, 0, *endNumber)
	for i := 0; i < *endNumber; i++ {
		var client *forestbus.Client
		if *enableDebug {
			client = forestbus.NewClient(clusterID, peers, forestbus.ClientEnableDebug())
		} else {
			client = forestbus.NewClient(clusterID, peers)
		}
		clients = append(clients, client)
	}

	for testnumber := *startNumber; testnumber <= *endNumber; testnumber += *stepSize {
		rowResults = append(rowResults, fmt.Sprintf("%v", testnumber))
		for testType := 0; testType < 3; testType++ {
			switch testType {
			case 0:
				numClients = testnumber
				batchSize = 1
			case 1:
				numClients = 1
				batchSize = testnumber
			case 2:
				numClients = testnumber
				batchSize = testnumber
			}

			runRes := make([]float64, 0, 3)

			for testrun := 0; testrun < 3; testrun++ {
				startTime := time.Now()
				for clientCount := 0; clientCount < numClients; clientCount++ {
					wg.Add(1)
					go clientSend(topic, &wg, clients[clientCount], *numBatches, batchSize, *messageSize, *msgTemplate, *waitForCommit)
				}
				wg.Wait()
				endTime := time.Now()

				totalMessages := numClients * *numBatches * batchSize
				totalBytes := float64(totalMessages * *messageSize)
				durationSeconds := endTime.Sub(startTime).Seconds()

				runRes = append(runRes, float64(totalMessages)/durationSeconds)
				rowResults = append(rowResults, fmt.Sprintf("%.f", float64(totalMessages)/durationSeconds))

				fmt.Printf("%v messages (%v) in %v [%v msgs/sec and %v/sec]\n", totalMessages, formatBytes(totalBytes), endTime.Sub(startTime), fmt.Sprintf("%.f", float64(totalMessages)/durationSeconds), formatBytes(float64(totalBytes)/durationSeconds))
			}
			avg := (runRes[0] + runRes[1] + runRes[2]) / 3
			rowResults = append(rowResults, fmt.Sprintf("%.f", avg))
			runRes = runRes[:0]
		}
		csvout.Write(rowResults)
		rowResults = rowResults[:0]
	}
	csvout.Flush()
	tout.Close()
	log.Printf("Done\n")
}
Esempio n. 4
0
func main() {
	// Initialise our configuration from flags
	var fbConnect = flag.String("forestbus", REQUIRED, "Connection string for Forest Bus cluster in ClusterID#Topic@host:port,host:port... format.")
	var enableDebug = flag.Bool("debug", false, "Enable debug logging")

	flag.Usage = PrintUsage

	flag.Parse()

	if flag.Lookup("help") != nil || flag.Lookup("h") != nil {
		flag.PrintDefaults()
		return
	}

	if *fbConnect == REQUIRED {
		fmt.Printf("forestbus connection string missing.\n")
		flag.PrintDefaults()
		return
	}

	clusterID, topic, peers := forestbus.ParseConnectionString(*fbConnect)

	if len(peers) == 0 {
		fmt.Printf("At least one node name is required.\n")
	}

	if topic == "" {
		fmt.Printf("topic missing.\n")
		flag.PrintDefaults()
		return
	}

	client := forestbus.NewClient(clusterID, peers)
	// Try up to 7 times (~14s) if there is a leadership change or network glitch
	batcher := forestbus.NewMessageBatcher(client, topic, forestbus.MessageBatcherRetries(forestbus.UNLIMITED_RETRIES))

	// Listen for signals
	exitSignal := make(chan os.Signal, 10)
	signal.Notify(exitSignal, os.Interrupt, os.Kill)

	// Setup the stdin reader
	reader := NewStdInReader()
	go reader.ReadStdIn()

	// Setup the message sender
	sender := NewMsgSender(batcher)
	go sender.Send(reader.MsgChan)
	var state State = RUNNING

	running := true
	if *enableDebug {
		log.Printf("Reading from stdin to send to topic %v\n", topic)
	}
	var sentCount, errCount int
	for running {
		select {
		case <-exitSignal:
			switch state {
			case RUNNING:
				// Ask the reader to stop generating new messages
				fmt.Printf("Exit detected - shutting down\n")
				reader.RequestStop()
				batcher.Close()
			case DRAINING:
				// Time to abort
				fmt.Printf("Abort detected - shutting down\n")
				sender.RequestStop()
				batcher.Close()
				state = ABORTING
			case COMPLETING:
				fmt.Printf("Abort detected - shutting down\n")
				state = ABORTING
			}
		case <-reader.FinishedChan:
			if state == RUNNING {
				state = DRAINING
			}
		case <-sender.FinishedChan:
			// Draining or running or abort has finished - all messages are in the MessageBatcher
			fmt.Printf("Messages sent - completing\n")
			state = COMPLETING
			batcher.Close()
			// Clear any final message stuck in Client
			client.Close()
			// Now we are done
			running = false
		case result := <-sender.ResultChan:
			if sendErr := result.GetError(); sendErr != nil {
				fmt.Printf("Error sending message: %v\n", sendErr)
				// Tell our stdin reader to stop reading and wait for it to confirm
				reader.RequestStop()
				errCount++
			} else {
				sentCount++
			}
		}
	}

	// Wait for all message results to come back
	waitToClear := true
	if (errCount + sentCount) == reader.Count {
		waitToClear = false
	}
	timeout := time.NewTimer(2 * time.Second)
	for waitToClear {
		select {
		case result := <-sender.ResultChan:
			if sendErr := result.GetError(); sendErr != nil {
				errCount++
			} else {
				sentCount++
			}
			if (errCount + sentCount) == reader.Count {
				waitToClear = false
			}
		case <-timeout.C:
			waitToClear = false
		}
	}

	fmt.Printf("Read %v messages, sent %v messages succesfully and %v errors.\n", reader.Count, sentCount, errCount)
}