func (s *ChecksumTestSuite) TestChecksum(t *C) {
var f string
// A most basic case
f = "hello world"
t.Check(
log.Checksum(f),
Equals,
"93CB22BB8F5ACDC3",
)
f = "select sourcetable, if(f.lastcontent = ?, f.lastupdate, f.lastcontent) as lastactivity, f.totalcount as activity, type.class as type, (f.nodeoptions & ?) as nounsubscribe from node as f inner join contenttype as type on type.contenttypeid = f.contenttypeid inner join subscribed as sd on sd.did = f.nodeid and sd.userid = ? union all select f.name as title, f.userid as keyval, ? as sourcetable, ifnull(f.lastpost, f.joindate) as lastactivity, f.posts as activity, ? as type, ? as nounsubscribe from user as f inner join userlist as ul on ul.relationid = f.userid and ul.userid = ? where ul.type = ? and ul.aq = ? order by title limit ?"
t.Check(
log.Checksum(f),
Equals,
"DB9EF18846547B8C",
)
}
func (w *SlowLogWorker) Run(job *Job) (*Result, error) {
w.status.Update(w.name, "Starting job "+job.Id)
result := &Result{}
// Open the slow log file.
file, err := os.Open(job.SlowLogFile)
if err != nil {
return nil, err
}
defer file.Close()
// Create a slow log parser and run it. It sends events log events via its channel.
// Be sure to stop it when done, else we'll leak goroutines. stopChan must be buffered
// so we don't block on send if parser crashes.
stopChan := make(chan bool, 1)
defer func() { stopChan <- true }()
opts := parser.Options{
StartOffset: uint64(job.StartOffset),
FilterAdminCommand: map[string]bool{
"Binlog Dump": true,
"Binlog Dump GTID": true,
},
}
p := parser.NewSlowLogParser(file, stopChan, opts)
go func() {
defer func() {
if r := recover(); r != nil {
errMsg := fmt.Sprintf("Error parsing %s: %s", job, r)
w.logger.Error(errMsg)
result.Error = errMsg
}
}()
p.Run()
}()
// The global class has info and stats for all events.
// Each query has its own class, defined by the checksum of its fingerprint.
global := mysqlLog.NewGlobalClass()
queries := make(map[string]*mysqlLog.QueryClass)
jobSize := job.EndOffset - job.StartOffset
var runtime time.Duration
var progress string
t0 := time.Now()
EVENT_LOOP:
for event := range p.EventChan {
runtime = time.Now().Sub(t0)
progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs",
float64(event.Offset)/float64(job.EndOffset)*100, event.Offset, job.EndOffset, jobSize, runtime.Seconds())
w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", job.SlowLogFile, progress))
// Check runtime, stop if exceeded.
if runtime >= job.RunTime {
errMsg := fmt.Sprintf("Timeout parsing %s: %s", progress)
w.logger.Warn(errMsg)
result.Error = errMsg
break EVENT_LOOP
}
if int64(event.Offset) >= job.EndOffset {
result.StopOffset = int64(event.Offset)
break EVENT_LOOP
}
// Add the event to the global class.
err := global.AddEvent(event)
switch err.(type) {
case mysqlLog.MixedRateLimitsError:
result.Error = err.Error()
break EVENT_LOOP
}
// Get the query class to which the event belongs.
fingerprint := mysqlLog.Fingerprint(event.Query)
classId := mysqlLog.Checksum(fingerprint)
class, haveClass := queries[classId]
if !haveClass {
class = mysqlLog.NewQueryClass(classId, fingerprint, job.ExampleQueries)
queries[classId] = class
}
// Add the event to its query class.
class.AddEvent(event)
}
w.status.Update(w.name, "Finalizing job "+job.Id)
if result.StopOffset == 0 {
result.StopOffset, _ = file.Seek(0, os.SEEK_CUR)
}
// Done parsing the slow log. Finalize the global and query classes (calculate
// averages, etc.).
for _, class := range queries {
class.Finalize()
}
global.Finalize(uint64(len(queries)))
// Sort the results, keep the top and combine the rest into a single class: Low-Ranking Queries (LRQ).
w.status.Update(w.name, "Combining LRQ job "+job.Id)
nQueries := len(queries)
classes := make([]*mysqlLog.QueryClass, nQueries)
for _, class := range queries {
// Decr before use; can't classes[--nQueries] in Go.
nQueries--
classes[nQueries] = class
}
result.Global = global
result.Classes = classes
if !job.ZeroRunTime {
result.RunTime = time.Now().Sub(t0).Seconds()
}
w.status.Update(w.name, "Done job "+job.Id)
w.logger.Info(fmt.Sprintf("Parsed %s: %s", job, progress))
return result, nil
}