func MakeReport(it proto.ServiceInstance, interval *Interval, result *Result, config Config) *Report { sort.Sort(ByQueryTime(result.Classes)) report := &Report{ ServiceInstance: it, StartTs: interval.StartTime, EndTs: interval.StopTime, SlowLogFile: interval.Filename, StartOffset: interval.StartOffset, EndOffset: interval.EndOffset, StopOffset: result.StopOffset, RunTime: result.RunTime, Global: result.Global, Class: result.Classes, } if config.ReportLimit == 0 { return report } n := len(result.Classes) if config.ReportLimit > 0 && n <= int(config.ReportLimit) { return report // no LRQ } // Top queries report.Class = result.Classes[0:config.ReportLimit] // Low-ranking Queries lrq := mysqlLog.NewQueryClass("0", "", false) for _, query := range result.Classes[config.ReportLimit:n] { addQuery(lrq, query) } report.Class = append(report.Class, lrq) return report }
func (w *SlowLogWorker) Run(job *Job) (*Result, error) { w.status.Update(w.name, "Starting job "+job.Id) result := &Result{} // Open the slow log file. file, err := os.Open(job.SlowLogFile) if err != nil { return nil, err } defer file.Close() // Create a slow log parser and run it. It sends events log events via its channel. // Be sure to stop it when done, else we'll leak goroutines. stopChan must be buffered // so we don't block on send if parser crashes. stopChan := make(chan bool, 1) defer func() { stopChan <- true }() opts := parser.Options{ StartOffset: uint64(job.StartOffset), FilterAdminCommand: map[string]bool{ "Binlog Dump": true, "Binlog Dump GTID": true, }, } p := parser.NewSlowLogParser(file, stopChan, opts) go func() { defer func() { if r := recover(); r != nil { errMsg := fmt.Sprintf("Error parsing %s: %s", job, r) w.logger.Error(errMsg) result.Error = errMsg } }() p.Run() }() // The global class has info and stats for all events. // Each query has its own class, defined by the checksum of its fingerprint. global := mysqlLog.NewGlobalClass() queries := make(map[string]*mysqlLog.QueryClass) jobSize := job.EndOffset - job.StartOffset var runtime time.Duration var progress string t0 := time.Now() EVENT_LOOP: for event := range p.EventChan { runtime = time.Now().Sub(t0) progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs", float64(event.Offset)/float64(job.EndOffset)*100, event.Offset, job.EndOffset, jobSize, runtime.Seconds()) w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", job.SlowLogFile, progress)) // Check runtime, stop if exceeded. if runtime >= job.RunTime { errMsg := fmt.Sprintf("Timeout parsing %s: %s", progress) w.logger.Warn(errMsg) result.Error = errMsg break EVENT_LOOP } if int64(event.Offset) >= job.EndOffset { result.StopOffset = int64(event.Offset) break EVENT_LOOP } // Add the event to the global class. err := global.AddEvent(event) switch err.(type) { case mysqlLog.MixedRateLimitsError: result.Error = err.Error() break EVENT_LOOP } // Get the query class to which the event belongs. fingerprint := mysqlLog.Fingerprint(event.Query) classId := mysqlLog.Checksum(fingerprint) class, haveClass := queries[classId] if !haveClass { class = mysqlLog.NewQueryClass(classId, fingerprint, job.ExampleQueries) queries[classId] = class } // Add the event to its query class. class.AddEvent(event) } w.status.Update(w.name, "Finalizing job "+job.Id) if result.StopOffset == 0 { result.StopOffset, _ = file.Seek(0, os.SEEK_CUR) } // Done parsing the slow log. Finalize the global and query classes (calculate // averages, etc.). for _, class := range queries { class.Finalize() } global.Finalize(uint64(len(queries))) // Sort the results, keep the top and combine the rest into a single class: Low-Ranking Queries (LRQ). w.status.Update(w.name, "Combining LRQ job "+job.Id) nQueries := len(queries) classes := make([]*mysqlLog.QueryClass, nQueries) for _, class := range queries { // Decr before use; can't classes[--nQueries] in Go. nQueries-- classes[nQueries] = class } result.Global = global result.Classes = classes if !job.ZeroRunTime { result.RunTime = time.Now().Sub(t0).Seconds() } w.status.Update(w.name, "Done job "+job.Id) w.logger.Info(fmt.Sprintf("Parsed %s: %s", job, progress)) return result, nil }