Example #1
0
func MakeReport(it proto.ServiceInstance, interval *Interval, result *Result, config Config) *Report {
	sort.Sort(ByQueryTime(result.Classes))

	report := &Report{
		ServiceInstance: it,
		StartTs:         interval.StartTime,
		EndTs:           interval.StopTime,
		SlowLogFile:     interval.Filename,
		StartOffset:     interval.StartOffset,
		EndOffset:       interval.EndOffset,
		StopOffset:      result.StopOffset,
		RunTime:         result.RunTime,
		Global:          result.Global,
		Class:           result.Classes,
	}

	if config.ReportLimit == 0 {
		return report
	}

	n := len(result.Classes)
	if config.ReportLimit > 0 && n <= int(config.ReportLimit) {
		return report // no LRQ
	}

	// Top queries
	report.Class = result.Classes[0:config.ReportLimit]

	// Low-ranking Queries
	lrq := mysqlLog.NewQueryClass("0", "", false)
	for _, query := range result.Classes[config.ReportLimit:n] {
		addQuery(lrq, query)
	}
	report.Class = append(report.Class, lrq)

	return report
}
Example #2
0
func (w *SlowLogWorker) Run(job *Job) (*Result, error) {
	w.status.Update(w.name, "Starting job "+job.Id)
	result := &Result{}

	// Open the slow log file.
	file, err := os.Open(job.SlowLogFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Create a slow log parser and run it.  It sends events log events via its channel.
	// Be sure to stop it when done, else we'll leak goroutines.  stopChan must be buffered
	// so we don't block on send if parser crashes.
	stopChan := make(chan bool, 1)
	defer func() { stopChan <- true }()
	opts := parser.Options{
		StartOffset: uint64(job.StartOffset),
		FilterAdminCommand: map[string]bool{
			"Binlog Dump":      true,
			"Binlog Dump GTID": true,
		},
	}
	p := parser.NewSlowLogParser(file, stopChan, opts)
	go func() {
		defer func() {
			if r := recover(); r != nil {
				errMsg := fmt.Sprintf("Error parsing %s: %s", job, r)
				w.logger.Error(errMsg)
				result.Error = errMsg
			}
		}()
		p.Run()
	}()

	// The global class has info and stats for all events.
	// Each query has its own class, defined by the checksum of its fingerprint.
	global := mysqlLog.NewGlobalClass()
	queries := make(map[string]*mysqlLog.QueryClass)
	jobSize := job.EndOffset - job.StartOffset
	var runtime time.Duration
	var progress string
	t0 := time.Now()

EVENT_LOOP:
	for event := range p.EventChan {
		runtime = time.Now().Sub(t0)
		progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs",
			float64(event.Offset)/float64(job.EndOffset)*100, event.Offset, job.EndOffset, jobSize, runtime.Seconds())
		w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", job.SlowLogFile, progress))

		// Check runtime, stop if exceeded.
		if runtime >= job.RunTime {
			errMsg := fmt.Sprintf("Timeout parsing %s: %s", progress)
			w.logger.Warn(errMsg)
			result.Error = errMsg
			break EVENT_LOOP
		}

		if int64(event.Offset) >= job.EndOffset {
			result.StopOffset = int64(event.Offset)
			break EVENT_LOOP
		}

		// Add the event to the global class.
		err := global.AddEvent(event)
		switch err.(type) {
		case mysqlLog.MixedRateLimitsError:
			result.Error = err.Error()
			break EVENT_LOOP
		}

		// Get the query class to which the event belongs.
		fingerprint := mysqlLog.Fingerprint(event.Query)
		classId := mysqlLog.Checksum(fingerprint)
		class, haveClass := queries[classId]
		if !haveClass {
			class = mysqlLog.NewQueryClass(classId, fingerprint, job.ExampleQueries)
			queries[classId] = class
		}

		// Add the event to its query class.
		class.AddEvent(event)
	}

	w.status.Update(w.name, "Finalizing job "+job.Id)

	if result.StopOffset == 0 {
		result.StopOffset, _ = file.Seek(0, os.SEEK_CUR)
	}

	// Done parsing the slow log.  Finalize the global and query classes (calculate
	// averages, etc.).
	for _, class := range queries {
		class.Finalize()
	}
	global.Finalize(uint64(len(queries)))

	// Sort the results, keep the top and combine the rest into a single class: Low-Ranking Queries (LRQ).
	w.status.Update(w.name, "Combining LRQ job "+job.Id)
	nQueries := len(queries)
	classes := make([]*mysqlLog.QueryClass, nQueries)
	for _, class := range queries {
		// Decr before use; can't classes[--nQueries] in Go.
		nQueries--
		classes[nQueries] = class
	}

	result.Global = global
	result.Classes = classes

	if !job.ZeroRunTime {
		result.RunTime = time.Now().Sub(t0).Seconds()
	}

	w.status.Update(w.name, "Done job "+job.Id)
	w.logger.Info(fmt.Sprintf("Parsed %s: %s", job, progress))
	return result, nil
}