func generateBenchmarkReport(dumpJSONFlag, dumpHTMLTarFlag, generatePlots bool, unitEngine *unit.UnitEngine) {
if dumpJSONFlag {
output.DumpJSON(unitEngine.Stats())
}
if dumpHTMLTarFlag {
html, err := output.Asset("output/embedded/render.html")
if err != nil {
log.Logger().Fatal(err)
}
scriptJs, err := output.Asset("output/embedded/script.js")
if err != nil {
log.Logger().Fatal(err)
}
output.DumpHTMLTar(html, scriptJs, unitEngine.Stats())
}
if generatePlots {
output.GeneratePlots(unitEngine.Stats(), runFlags.verbose)
}
output.PrintReport(unitEngine.Stats(), os.Stderr)
}
func (e *UnitEngine) stopUnit(id string, state UnitState) {
e.mu.Lock()
newState := state
newState.stopRequestTime = time.Now()
if Verbose {
log.Logger().Infof("marking unit as to be deleted: %s", id)
}
e.stoppingUnits[id] = newState
e.mu.Unlock()
err := e.StopFunc(id)
if err != nil {
log.Logger().Warning(err)
}
}
func generateDelayStartPlot(fname string, persist bool, debug bool, plotsDirectory string, stats unit.Stats) {
for _, process := range processTypes {
p, err := gnuplot.NewPlotter(fname, persist, debug)
if err != nil {
err_string := fmt.Sprintf("** err: %v\n", err)
panic(err_string)
}
defer p.Close()
for hostname, metrics := range stats.MachineStats {
f, err1 := os.Create(fmt.Sprintf("%s/%s-%s.dat", plotsDirectory, process, hostname))
if err1 != nil {
err_string := fmt.Sprintf("** err: %v\n", err1)
panic(err_string)
}
defer f.Close()
valuesX := make([]float64, 0)
valuesY := make([]float64, 0)
for _, metric := range metrics {
if metric.Process == process {
f.WriteString(fmt.Sprintf("%v %v\n", metric.TimeStamp, metric.CPUUsage))
valuesX = append(valuesX, metric.TimeStamp)
valuesY = append(valuesY, metric.CPUUsage)
}
}
if debug {
log.Logger().Infof("Plotting data for %s", hostname)
}
f.Sync()
p.PlotXY(valuesX, valuesY, fmt.Sprintf("%s - Time/CPU", hostname), "")
}
p.SetXLabel("Timestamp (secs)")
p.SetYLabel("CPU usage (per)")
p.CheckedCmd("set terminal pdf")
if debug {
log.Logger().Infof("Generating plot %s", process)
}
p.CheckedCmd(fmt.Sprintf("set output '%s/%s.pdf'", plotsDirectory, process))
p.CheckedCmd("replot")
time.Sleep(2)
p.CheckedCmd("q")
}
}
func (s *UnitObserver) HelloHandler(unitID string, w http.ResponseWriter, r *http.Request) {
delay := s.unitEngine.MarkUnitRunning(unitID)
if Verbose {
log.Logger().Infof("marked unit as running: %s [%d] %f", unitID, len(s.unitEngine.runningUnits), delay.Seconds())
}
w.Write([]byte("ok.\n"))
}
func (s *UnitObserver) ByeHandler(unitID string, w http.ResponseWriter, r *http.Request) {
if Verbose {
log.Logger().Infof("marking unit as stopped: %s [%d]", unitID, len(s.unitEngine.stoppedUnits)+len(s.unitEngine.runningUnits)+len(s.unitEngine.startingUnits))
}
s.unitEngine.MarkUnitStopped(unitID)
w.Write([]byte("ok.\n"))
}
// BenchmarkDefByFile procudes a benchmark definition out of a YAML file
// Return a benchmark definition object and error
func BenchmarkDefByFile(filePath string) (BenchmarkDef, error) {
def, err := parseBenchmarkDef(filePath)
if err != nil {
log.Logger().Fatalf("error when parsing the benchmark definition %v", err)
}
return def, err
}
func (e *UnitEngine) float(obj definition.Float) {
if Verbose {
log.Logger().Info("float instruction not implemented yet...")
}
//duration := time.Duration(obj.Duration) * time.Millisecond
//log.Infof("floating for %s\n", duration)
//time.Sleep(duration)
}
func parseBenchmarkDef(filePath string) (BenchmarkDef, error) {
filename, _ := filepath.Abs(filePath)
yamlFile, err := ioutil.ReadFile(filename)
if err != nil {
log.Logger().Fatalf("unable to read yaml file %v", err)
}
def := BenchmarkDef{}
if err := yaml.Unmarshal(yamlFile, &def); err != nil {
log.Logger().Fatalf("unable to parse the yaml test definition: %v", err)
}
if !validateDefinition(def) {
log.Logger().Fatal("benchmark definition contains wrong values")
}
return def, nil
}
func genRandomID() string {
c := 5
b := make([]byte, c)
_, err := rand.Read(b)
if err != nil {
log.Logger().Fatal(err)
}
return hex.EncodeToString(b)
}
func (s *UnitObserver) StartHTTPService(addr string) {
r := mux.NewRouter()
r.HandleFunc("/hello/{unitID}", withIDParam(s.HelloHandler)).Methods("GET")
r.HandleFunc("/alive/{unitID}", withIDParam(s.AliveHandler)).Methods("GET")
r.HandleFunc("/bye/{unitID}", withIDParam(s.ByeHandler)).Methods("GET")
r.HandleFunc("/stats/{statsID}", s.StatsHandler).Methods("POST")
http.Handle("/", r)
if Verbose {
log.Logger().Infof("listening on %s\n", addr)
}
go http.ListenAndServe(addr, nil)
if Verbose {
log.Logger().Infof("listening on %s\n", addr)
}
}
func (b *Builder) UseCustomUnitFileService(filePath string) error {
filename, _ := filepath.Abs(filePath)
unitFile, err := ioutil.ReadFile(filename)
if err != nil {
log.Logger().Errorf("unable to read yaml file %v", err)
return err
}
contents := string(unitFile)
if strings.Contains(contents, "Global=true") {
log.Logger().Warningf("Global fleet scheduling option 'Global=true' can cause undesired results")
}
b.unitFile, err = unit.NewUnitFile(contents)
if err != nil {
log.Logger().Errorf("error creating Unit from %q: %v", contents, err)
return err
}
return nil
}
func withIDParam(handler func(unitID string, w http.ResponseWriter, r *http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
unitID := mux.Vars(r)["unitID"]
if unitID == "" {
log.Logger().Error("empty unitID")
w.WriteHeader(400)
} else {
handler(unitID, w, r)
}
}
}
func (f runCmdFlags) Validate() {
if f.dumpJSONFlag && f.dumpHTMLTarFlag {
log.Logger().Fatal("dump option is required. Please, choose between: dump-json OR dump-html-tar")
}
if f.generatePlots {
if _, err := exec.LookPath("gnuplot"); err != nil {
log.Logger().Infof("generate-gnuplots: could not find path to 'gnuplot':\n%v\n", err)
log.Logger().Fatal("generate-gnuplots option requires 'gnuplot' software installed")
}
}
if f.benchmarkFile == "" && f.rawInstructions == "" {
log.Logger().Fatal("benchmark file definition or raw instructions is required")
}
if f.benchmarkFile != "" && f.rawInstructions != "" {
log.Logger().Fatal("benchmark file definition or raw instructions are mutual exclusive")
}
if f.benchmarkFile == "" && f.rawInstructions != "" && f.igSize <= 0 {
log.Logger().Fatal("instance group size has to be greater than 0 when using raw-instructions parameter")
}
}
func validateDefinition(benchmark BenchmarkDef) bool {
if benchmark.InstanceGroupSize <= 0 {
log.Logger().Fatal("instance group size has to be greater or equal to 1")
}
// Validate application definition
if benchmark.Application.Type != "" && benchmark.Application.Type != "unitfiles" && benchmark.Application.Type != "docker" && benchmark.Application.Type != "rkt" {
log.Logger().Errorf("wrong application type %v", benchmark.Application.Type)
return false
}
if benchmark.Application.Type == "unitfiles" && benchmark.Application.UnitFilePath == "" {
log.Logger().Errorf("application unit file path is required for type %v", benchmark.Application.Type)
return false
}
if benchmark.Application.Image == "" && benchmark.Application.Type == "docker" && benchmark.Application.Type == "rkt" {
log.Logger().Warning("application image is empty using standard container")
}
emptyInstruction := &Instruction{}
for _, instruction := range benchmark.Instructions {
if instruction.Start != emptyInstruction.Start && (instruction.Start.Max == 0 || instruction.Start.Interval < 0) {
log.Logger().Errorf("wrong values for the start operation of instruction %v", instruction)
return false
}
if instruction.Float != emptyInstruction.Float && (instruction.Float.Rate <= 0 || instruction.Float.Duration <= 0) {
log.Logger().Errorf("wrong values for the start operation of instruction %v", instruction)
return false
}
if instruction.ExpectRunning != emptyInstruction.ExpectRunning &&
((instruction.ExpectRunning.Symbol != Lower && instruction.ExpectRunning.Symbol != Greater) || instruction.ExpectRunning.Amount < 0) {
log.Logger().Errorf("wrong values for the expect-running operation of instruction %v", instruction)
return false
}
if instruction.Sleep < 0 {
log.Logger().Errorf("wrong values for the sleep operation of instruction %v", instruction)
return false
}
if instruction.Stop != "" && instruction.Stop != StopAll {
log.Logger().Errorf("wrong values for the stop operation of instruction %v", instruction)
return false
}
}
return true
}
// MarkUnitStopped collects the timestamps of the stop operation for an unit
func (e *UnitEngine) MarkUnitStopped(id string) {
e.mu.Lock()
defer e.mu.Unlock()
state, exists := e.stoppingUnits[id]
if !exists {
log.Logger().Errorf("unit %s cannot be found in the stopping pool\n", id)
return
}
delete(e.stoppingUnits, id)
state.actualStopTime = time.Now()
e.stoppedUnits[id] = state
e.stoppedStats = append(e.stoppedStats, e.genStatsLine(id, state.actualStopTime.Sub(state.stopRequestTime)))
}
// MarkUnitRunning collects the timestamps of the start operation for an unit
func (e *UnitEngine) MarkUnitRunning(id string) time.Duration {
e.mu.Lock()
defer e.mu.Unlock()
state, exists := e.startingUnits[id]
if !exists {
log.Logger().Errorf("unit %s cannot be found in the starting pool\n", id)
return time.Duration(0)
}
delete(e.startingUnits, id)
state.actualStartTime = time.Now()
e.runningUnits[id] = state
e.startedStats = append(e.startedStats, e.genStatsLine(id, state.actualStartTime.Sub(state.startRequestTime)))
return state.actualStartTime.Sub(state.startRequestTime)
}
// DumpJSON dumps the stats metrics to a javascript file 'data.js' which should
// be used by embedded scripts to print a graphic.
func DumpHTMLTar(html []byte, scriptJs []byte, stats unit.Stats) {
jsonData := bytes.NewBufferString("var allData = ")
enc := json.NewEncoder(jsonData)
enc.Encode(stats)
jsonData.WriteString(";\n")
tw := tar.NewWriter(os.Stdout)
var files = []struct {
Name string
Body []byte
}{
{"data.js", jsonData.Bytes()},
{"index.html", html},
{"script.js", scriptJs},
}
for _, file := range files {
hdr := &tar.Header{
Name: file.Name,
ChangeTime: time.Now(),
ModTime: time.Now(),
Mode: 0644,
Size: int64(len(file.Body)),
}
if err := tw.WriteHeader(hdr); err != nil {
log.Logger().Fatal(err)
}
if _, err := tw.Write([]byte(file.Body)); err != nil {
log.Logger().Fatal(err)
}
}
if err := tw.Close(); err != nil {
log.Logger().Fatal(err)
}
}
func (s *UnitObserver) StatsHandler(w http.ResponseWriter, r *http.Request) {
statsID := mux.Vars(r)["statsID"]
b := bytes.NewBufferString("")
b.ReadFrom(r.Body)
var hostname string
var cpuusage float64
var rss int
n, err := fmt.Sscanf(b.String(), "%s %f %d", &hostname, &cpuusage, &rss)
if err != nil || n != 3 {
log.Logger().Warningf("don't know how to parse statsline: " + b.String())
return
}
s.unitEngine.DumpProcessStats(statsID, hostname, cpuusage, rss)
}
// BenchmarkDefByRawInstructions creates a benchmark definition using raw
// instructions and instance group size
// Return a benchmark definition and error
func BenchmarkDefByRawInstructions(instructions string, igSize int) (BenchmarkDef, error) {
re := regexp.MustCompile(`\(([^\)]+)\)`)
parsed := re.FindAllStringSubmatch(instructions, -1)
def := BenchmarkDef{}
def.Instructions = make([]Instruction, 0)
def.InstanceGroupSize = igSize
for _, cmdString := range parsed {
if len(cmdString) < 2 {
continue
}
splitted := strings.Fields(cmdString[1])
if len(splitted) == 0 {
continue
}
cmd := splitted[0]
args := []string{}
if len(splitted) > 1 {
args = splitted[1:]
}
switch cmd {
case instructionStart:
if len(args) != 2 {
log.Logger().Fatalf("start requires 2 arguments: max and time between starts. eg: (start 10 100ms)")
}
max, err := strconv.Atoi(args[0])
if err != nil {
log.Logger().Fatalf("%v", err)
}
var interval int
interval, err = strconv.Atoi(args[1])
if err != nil {
log.Logger().Fatalf("%v", err)
}
def.Instructions = append(def.Instructions, Instruction{
Start: Start{
Max: max,
Interval: interval,
},
})
break
case instructionFloat:
if len(args) != 2 {
log.Logger().Fatalf("float requires 2 arguments: rate and duration")
}
rate, err := strconv.ParseFloat(args[0], 64)
if err != nil {
log.Logger().Fatalf("%v", err)
}
var duration int
duration, err = strconv.Atoi(args[1])
if err != nil {
log.Logger().Fatalf("%v", err)
}
def.Instructions = append(def.Instructions, Instruction{
Float: Float{
Rate: rate,
Duration: duration,
},
})
break
case instructionSleep:
timeout, err := strconv.Atoi(args[0])
if err != nil {
log.Logger().Fatalf("%v", err)
}
def.Instructions = append(def.Instructions, Instruction{
Sleep: timeout,
})
break
case instructionExpectRunning:
if len(args) != 2 {
log.Logger().Fatal("expect-running requires 2 arguments: [><] int")
}
qty, err := strconv.Atoi(args[1])
if err != nil {
log.Logger().Fatalf("%v", err)
}
symbol := ExpectRunningSymbol(args[0])
if symbol != Lower && symbol != Greater {
log.Logger().Fatalf("expect-running comparator has to be > or <")
}
def.Instructions = append(def.Instructions, Instruction{
ExpectRunning: ExpectRunning{
Symbol: symbol,
Amount: qty,
},
})
break
case "stop-all":
def.Instructions = append(def.Instructions, Instruction{
Stop: StopCommand(cmd),
})
break
}
}
return def, nil
}
func runRun(cmd *cobra.Command, args []string) {
runFlags.Validate()
fleetPool := fleet.NewFleetPool(20)
if runFlags.listenAddr == "" {
// We extract the public CoreOS ip of the host machine
ip, err := fleet.CoreosHostPublicIP()
if ip == "" || err != nil {
runFlags.listenAddr = listenerDefaultIP + ":" + listenerDefaultPort
} else {
runFlags.listenAddr = ip + ":" + listenerDefaultPort
}
}
existingUnits, err := fleetPool.ListUnits()
if err != nil {
log.Logger().Fatal(err)
}
var benchmark definition.BenchmarkDef
if runFlags.benchmarkFile == "" {
benchmark, err = definition.BenchmarkDefByRawInstructions(runFlags.rawInstructions, runFlags.igSize)
if err != nil {
log.Logger().Fatal("unable to parse the introduced raw instructions")
}
} else {
benchmark, err = definition.BenchmarkDefByFile(runFlags.benchmarkFile)
}
if err != nil {
log.Logger().Fatal(err)
}
unitEngine, err := unit.NewEngine(benchmark, runFlags.verbose)
if err != nil {
log.Logger().Fatal(err)
}
builder, err := unit.NewBuilder(benchmark.Application, unitEngine.InstanceGroupSize(), runFlags.listenAddr)
if err != nil {
log.Logger().Fatal(err)
}
if benchmark.Application.Type == "unitfiles" {
err = builder.UseCustomUnitFileService(benchmark.Application.UnitFilePath)
if err != nil {
log.Logger().Fatal("unable to parse unit file from your application definition")
}
}
observer := unit.NewUnitObserver(unitEngine)
observer.StartHTTPService(runFlags.listenAddr)
fleetPool.StartUnit(builder.MakeStatsDumper("etcd", "echo `hostname` `docker run --rm --pid=host ragnarb/toolbox pidstat -h -r -u -C etcd 10 1 | tail -n 1 | awk \\'{print $7 \" \" $12}\\'`", "etcd"))
fleetPool.StartUnit(builder.MakeStatsDumper("fleetd", "echo `hostname` `docker run --rm --pid=host ragnarb/toolbox pidstat -h -r -u -C fleetd 10 1 | tail -n 1 | awk \\'{print $7 \" \" $12}\\'`", "fleetd"))
fleetPool.StartUnit(builder.MakeStatsDumper("systemd", "echo `hostname` `docker run --rm --pid=host ragnarb/toolbox pidstat -h -r -u -p 1 10 1 | tail -n 1 | awk \\'{print $7 \" \" $12}\\'`", "systemd"))
unitEngine.SpawnFunc = func(id string) error {
if runFlags.verbose {
log.Logger().Infof("spawning unit with id %s\n", id)
}
return fleetPool.StartUnitGroup(builder.MakeUnitChain(id))
}
unitEngine.StopFunc = func(id string) error {
if runFlags.verbose {
log.Logger().Infof("stopping unit with id %s\n", id)
}
return fleetPool.Stop(builder.GetUnitPrefix() + "-0@" + id + ".service")
}
unitEngine.Run()
existingUnits, err = fleetPool.ListUnits()
if err != nil {
log.Logger().Errorf("error listing units %v", err)
}
wg := new(sync.WaitGroup)
for _, unit := range existingUnits {
if strings.HasPrefix(unit.Name, builder.GetUnitPrefix()) {
wg.Add(1)
go func(unitName string) {
if runFlags.verbose {
log.Logger().Infof("destroying old unit: %s", unitName)
}
fleetPool.Destroy(unitName)
wg.Done()
}(unit.Name)
}
}
wg.Wait()
generateBenchmarkReport(runFlags.dumpJSONFlag, runFlags.dumpHTMLTarFlag, runFlags.generatePlots, unitEngine)
}