// startCtfeMetrics registers gauges with the graphite server that indicate CT is running healthily
// and starts a goroutine to update them periodically.
func startCtfeMetrics() {
pendingTasksGauge := metrics.GetOrRegisterGauge("num-pending-tasks", metrics.DefaultRegistry)
oldestPendingTaskAgeGauge := metrics.GetOrRegisterGaugeFloat64("oldest-pending-task-age", metrics.DefaultRegistry)
// 0=no tasks pending; 1=started; 2=not started
oldestPendingTaskStatusGauge := metrics.GetOrRegisterGauge("oldest-pending-task-status", metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(common.SAMPLE_PERIOD) {
pendingTaskCount, err := pending_tasks.GetPendingTaskCount()
if err != nil {
glog.Error(err)
} else {
pendingTasksGauge.Update(pendingTaskCount)
}
oldestPendingTask, err := pending_tasks.GetOldestPendingTask()
if err != nil {
glog.Error(err)
} else if oldestPendingTask == nil {
oldestPendingTaskAgeGauge.Update(0)
oldestPendingTaskStatusGauge.Update(0)
} else {
addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(oldestPendingTask.GetCommonCols().TsAdded.Int64, 10))
oldestPendingTaskAgeGauge.Update(time.Since(addedTime).Seconds())
if oldestPendingTask.GetCommonCols().TsStarted.Valid {
oldestPendingTaskStatusGauge.Update(1)
} else {
oldestPendingTaskStatusGauge.Update(2)
}
}
}
}()
}
func MakeRules(cfgFile string, dbClient *influxdb.Client, tickInterval time.Duration, am *alerting.AlertManager, testing bool) ([]*Rule, error) {
parsedRules, err := parseAlertRules(cfgFile)
if err != nil {
return nil, err
}
rules := map[string]*Rule{}
for _, r := range parsedRules {
r, err := newRule(r, dbClient, testing, tickInterval)
if err != nil {
return nil, err
}
if _, ok := rules[r.Name]; ok {
return nil, fmt.Errorf("Found multiple rules with the same name: %s", r.Name)
}
rules[r.Name] = r
}
// Start the goroutines.
rv := make([]*Rule, 0, len(rules))
for _, r := range rules {
rv = append(rv, r)
go func(rule *Rule) {
if err := rule.tick(am); err != nil {
glog.Error(err)
}
for _ = range time.Tick(tickInterval) {
if err := rule.tick(am); err != nil {
glog.Error(err)
}
}
}(r)
}
return rv, nil
}
func pollAndExecOnce() {
pending, err := frontend.GetOldestPendingTaskV2()
if err != nil {
glog.Error(err)
return
}
task := asPollerTask(pending)
if task == nil {
return
}
taskName, id := task.GetTaskName(), task.GetCommonCols().Id
glog.Infof("Preparing to execute task %s %d", taskName, id)
if err = updateAndBuild(); err != nil {
glog.Error(err)
return
}
glog.Infof("Executing task %s %d", taskName, id)
if err = task.Execute(); err == nil {
glog.Infof("Completed task %s %d", taskName, id)
} else {
glog.Errorf("Task %s %d failed: %v", taskName, id, err)
if !*dryRun {
if err := updateWebappTaskSetFailed(task); err != nil {
glog.Error(err)
}
}
}
}
func doWorkerHealthCheck() {
if err := updateAndBuild(); err != nil {
glog.Error(err)
return
}
if err := checkWorkerHealth(); err != nil {
glog.Error(err)
return
}
}
// Search returns a slice of Issues which fit the given criteria.
func (r Rietveld) Search(limit int, terms ...*SearchTerm) ([]*Issue, error) {
searchUrl := fmt.Sprintf("/search?format=json&limit=%d", limit)
for _, term := range terms {
searchUrl += fmt.Sprintf("&%s=%s", term.Key, term.Value)
}
var issues issueListSortable
cursor := ""
for {
var data rietveldResults
err := r.get(searchUrl+cursor, &data)
if err != nil {
return nil, fmt.Errorf("Rietveld search failed: %v", err)
}
if len(data.Results) == 0 {
break
}
for _, issue := range data.Results {
fullIssue, err := r.getIssueProperties(issue.Issue, true)
if err != nil {
glog.Error(err)
} else {
fullIssue.Created = parseTime(fullIssue.CreatedString)
fullIssue.Modified = parseTime(fullIssue.ModifiedString)
for _, msg := range fullIssue.Messages {
committed := false
for _, r := range committedIssueRegexp {
committed, err = regexp.MatchString(r, msg.Text)
if committed {
break
}
}
msg.Date = parseTime(msg.DateString)
if err != nil {
glog.Error(err)
continue
}
if committed {
fullIssue.Committed = true
}
}
issues = append(issues, &fullIssue)
}
}
if len(issues) >= limit {
break
}
cursor = "&cursor=" + data.Cursor
}
sort.Sort(issues)
return issues, nil
}
// loop runs the AlertManager's main loop.
func (am *AlertManager) loop() {
if err := am.tick(); err != nil {
glog.Error(err)
}
for _ = range time.Tick(am.tickInterval) {
select {
case <-am.interrupt:
return
default:
}
if err := am.tick(); err != nil {
glog.Error(err)
}
}
}
// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(git *gitinfo.GitInfo, tileDir, datasetName string, ri ingester.ResultIngester, config map[string]string, every time.Duration, nCommits int, minDuration time.Duration, statusDir, metricName string) ProcessStarter {
return func() {
i, err := ingester.NewIngester(git, tileDir, datasetName, ri, nCommits, minDuration, config, statusDir, metricName)
if err != nil {
glog.Fatalf("Failed to create Ingester: %s", err)
}
glog.Infof("Starting %s ingester. Run every %s.", datasetName, every.String())
// oneStep is a single round of ingestion.
oneStep := func() {
glog.Infof("Running ingester: %s", datasetName)
err := i.Update()
if err != nil {
glog.Error(err)
}
glog.Infof("Finished running ingester: %s", datasetName)
}
// Start the ingester.
go func() {
oneStep()
for _ = range time.Tick(every) {
oneStep()
}
}()
}
}
func NewPollingStatus(value interface{}, poll func(interface{}) error, frequency time.Duration) (*PollingStatus, error) {
s := PollingStatus{
sync.RWMutex{},
value,
poll,
make(chan bool),
}
if err := s.poll(); err != nil {
return nil, err
}
go func(s *PollingStatus) {
ticker := time.Tick(frequency)
for {
select {
case <-s.stop:
return
case <-ticker:
if err := s.poll(); err != nil {
glog.Error(err)
}
}
}
}(&s)
return &s, nil
}
func main() {
defer common.LogPanic()
common.Init()
frontend.MustInit()
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&admin_tasks.RecreateWebpageArchivesUpdateVars{}, *gaeTaskID))
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Capture archives", util.GetMasterLogLink(*runID), ""))
// Ensure webapp is updated and completion email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Capture archives on Workers")
defer glog.Flush()
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
cmd := []string{
fmt.Sprintf("cd %s;", util.CtTreeDir),
"git pull;",
"make all;",
// The main command that runs capture_archives on all workers.
fmt.Sprintf("DISPLAY=:0 capture_archives --worker_num=%s --log_dir=%s --log_id=%s --pageset_type=%s --chromium_build=%s;", util.WORKER_NUM_KEYWORD, util.GLogDir, *runID, *pagesetType, *chromiumBuild),
}
_, err := util.SSH(strings.Join(cmd, " "), util.Slaves, util.CAPTURE_ARCHIVES_TIMEOUT)
if err != nil {
glog.Errorf("Error while running cmd %s: %s", cmd, err)
return
}
*taskCompletedSuccessfully = true
}
func statusJsonHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// Obtain the status info. Only display error messages if the user
// is a logged-in Googler.
status := arb.GetStatus(login.IsAGoogler(r))
if err := json.NewEncoder(w).Encode(&status); err != nil {
glog.Error(err)
}
}
func main() {
defer common.LogPanic()
common.Init()
if *emailTokenPath == "" {
glog.Error("Must specify --email_token_path")
return
}
defer glog.Flush()
for _, shiftType := range allShiftTypes {
res, err := http.Get(shiftType.nextSheriffEndpoint)
if err != nil {
glog.Fatalf("Could not HTTP Get: %s", err)
}
defer util.Close(res.Body)
var jsonType map[string]interface{}
if err := json.NewDecoder(res.Body).Decode(&jsonType); err != nil {
glog.Fatalf("Could not unmarshal JSON: %s", err)
}
sheriffEmail, _ := jsonType["username"].(string)
if sheriffEmail == NO_SHERIFF {
glog.Infof("Skipping emailing %s because %s was specified", shiftType.shiftName, NO_SHERIFF)
continue
}
sheriffUsername := strings.Split(string(sheriffEmail), "@")[0]
emailTemplateParsed := template.Must(template.New("sheriff_email").Parse(EMAIL_TEMPLATE))
emailBytes := new(bytes.Buffer)
if err := emailTemplateParsed.Execute(emailBytes, struct {
SheriffName string
SheriffType string
SheriffSchedules string
SheriffDoc string
ScheduleStart string
ScheduleEnd string
}{
SheriffName: sheriffUsername,
SheriffType: shiftType.shiftName,
SheriffSchedules: shiftType.schedulesLink,
SheriffDoc: shiftType.documentationLink,
ScheduleStart: jsonType["schedule_start"].(string),
ScheduleEnd: jsonType["schedule_end"].(string),
}); err != nil {
glog.Errorf("Failed to execute template: %s", err)
return
}
emailSubject := fmt.Sprintf("%s is the next %s", sheriffUsername, shiftType.shiftName)
if err := sendEmail([]string{sheriffEmail, EXTRA_RECIPIENT}, emailSubject, emailBytes.String()); err != nil {
glog.Fatalf("Error sending email to sheriff: %s", err)
}
}
}
// FileExists returns true if the given path exists and false otherwise.
// If there is an error it will return false and log the error message.
func FileExists(path string) bool {
if _, err := os.Stat(path); os.IsNotExist(err) {
return false
} else if err != nil {
glog.Error(err)
return false
}
return true
}
func main() {
common.Init()
webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.RecreatePageSetsUpdateVars{}, *gaeTaskID))
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Creating pagesets"))
// Ensure webapp is updated and completion email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Creating Pagesets on Workers")
defer glog.Flush()
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
cmd := []string{
fmt.Sprintf("cd %s;", util.CtTreeDir),
"git pull;",
"make all;",
// The main command that runs create_pagesets on all workers.
fmt.Sprintf("create_pagesets --worker_num=%s --log_dir=%s --pageset_type=%s;", util.WORKER_NUM_KEYWORD, util.GLogDir, *pagesetType),
}
// Setting a 4 hour timeout since it may take a while to upload page sets to
// Google Storage when doing 10k page sets per worker.
if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 4*time.Hour); err != nil {
glog.Errorf("Error while running cmd %s: %s", cmd, err)
return
}
*taskCompletedSuccessfully = true
}
func main() {
defer common.LogPanic()
master_common.Init()
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&admin_tasks.RecreatePageSetsUpdateVars{}, *gaeTaskID))
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Creating pagesets", util.GetMasterLogLink(*runID), ""))
// Ensure webapp is updated and completion email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
if !*master_common.Local {
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
}
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Creating Pagesets on Workers")
defer glog.Flush()
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
cmd := append(master_common.WorkerSetupCmds(),
// The main command that runs create_pagesets on all workers.
fmt.Sprintf(
"create_pagesets --worker_num=%s --log_dir=%s --log_id=%s --pageset_type=%s --local=%t;",
util.WORKER_NUM_KEYWORD, util.GLogDir, *runID, *pagesetType, *master_common.Local))
_, err := util.SSH(strings.Join(cmd, " "), util.Slaves, util.CREATE_PAGESETS_TIMEOUT)
if err != nil {
glog.Errorf("Error while running cmd %s: %s", cmd, err)
return
}
*taskCompletedSuccessfully = true
}
func mainHandler(w http.ResponseWriter, r *http.Request) {
if _, err := w.Write([]byte(`
<html>
<body>
Hello World
</body>
</html>
`)); err != nil {
glog.Error(err)
}
}
// LastSkpCommit returns the time of the last change to the SKP_VERSION file.
func (g *GitInfo) LastSkpCommit() (time.Time, error) {
// Executes a git log command that looks like:
//
// git log --format=format:%ct -n 1 SKP_VERSION
//
// The output should be a single unix timestamp.
cmd := exec.Command("git", "log", "--format=format:%ct", "-n", "1", "SKP_VERSION")
cmd.Dir = g.dir
b, err := cmd.Output()
if err != nil {
glog.Error("Failed to read git log: ", err)
return time.Time{}, err
}
ts, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
glog.Error("Failed to parse timestamp: ", string(b), err)
return time.Time{}, err
}
return time.Unix(ts, 0), nil
}
func rulesJsonHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
rules := struct {
Rules []*rules.Rule `json:"rules"`
}{
Rules: rulesList,
}
if err := json.NewEncoder(w).Encode(&rules); err != nil {
glog.Error(err)
}
}
// ReadLinesFromPipe reads from the given pipe and logs its output.
func ReadLinesFromPipe(pipe io.Reader, lines chan string) {
buf := []byte{}
// writeLine recovers from a panic when writing to the channel.
writeLine := func(s string) {
defer func() {
if r := recover(); r != nil {
glog.Warningf("Panic writing to channel... are we exiting?")
}
}()
lines <- s
}
// readLines reads output lines from the buffer and pushes them into the channel.
readlines := func() {
readIdx := 0
// Read lines from buf.
for i, b := range buf {
if b == '\n' {
s := string(buf[readIdx:i])
writeLine(s)
readIdx = i + 1
}
}
// Remove the lines we read.
buf = buf[readIdx:]
}
// Loop forever, reading bytes from the pipe.
readbuf := make([]byte, 4096)
for {
read, err := pipe.Read(readbuf)
if read > 0 {
buf = append(buf, readbuf[:read]...)
// Read lines from the buffers.
readlines()
}
if err != nil {
if err == io.EOF {
break
} else {
glog.Error(err)
}
} else if read == 0 {
time.Sleep(20 * time.Millisecond)
}
}
// Read any remaining lines from the buffers.
readlines()
// "Flush" any incomplete lines from the buffers.
writeLine(string(buf))
}
func main() {
defer common.LogPanic()
master_common.Init()
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&chromium_builds.UpdateVars{}, *gaeTaskID))
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Build chromium", util.GetMasterLogLink(*runID), ""))
// Ensure webapp is updated and completion email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
if !*master_common.Local {
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
}
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Running build chromium")
defer glog.Flush()
if *chromiumHash == "" {
glog.Error("Must specify --chromium_hash")
return
}
if *skiaHash == "" {
glog.Error("Must specify --skia_hash")
return
}
if _, _, err := util.CreateChromiumBuild("", *targetPlatform, *chromiumHash, *skiaHash, *applyPatches); err != nil {
glog.Errorf("Error while creating the Chromium build: %s", err)
return
}
taskCompletedSuccessfully = true
}
// JsonHandler is a pre-built handler for HTTP requests which returns version
// information in JSON format.
func JsonHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
v, err := GetVersion()
if err != nil {
glog.Error(err)
v = &Version{
Commit: "(unknown)",
Date: time.Time{},
}
}
if err := json.NewEncoder(w).Encode(v); err != nil {
glog.Errorf("Failed to write or encode output: %s", err)
return
}
}
// Runs commonly-used initialization metrics.
func Init() {
flag.Parse()
defer glog.Flush()
flag.VisitAll(func(f *flag.Flag) {
glog.Infof("Flags: --%s=%v", f.Name, f.Value)
})
// See skbug.com/4386 for details on why the below section exists.
glog.Info("Initializing logserver for log level INFO.")
glog.Warning("Initializing logserver for log level WARNING.")
glog.Error("Initializing logserver for log level ERROR.")
// Use all cores.
runtime.GOMAXPROCS(runtime.NumCPU())
}
// reportError serves the error page with the given message.
func reportError(w http.ResponseWriter, msg string, code int) {
errData := struct {
Code int
CodeString string
Message string
}{
Code: code,
CodeString: http.StatusText(code),
Message: msg,
}
w.WriteHeader(code)
err := templates.ExecuteTemplate(w, "error.html", errData)
if err != nil {
glog.Error("Failed to display error.html!!")
}
}
// newDocSet does the core of the work for both NewDocSet and NewDocSetForIssue.
//
// The repo is checked out into repoDir.
// If a valid issue and patchset are supplied then the repo will be patched with that CL.
// If refresh is true then the git repo will be periodically refreshed (git pull).
func newDocSet(repoDir, repo string, issue, patchset int64, refresh bool) (*DocSet, error) {
if issue > 0 {
issueInfo, err := rc.Issue(issue)
if err != nil {
err := fmt.Errorf("Failed to retrieve issue status %d: %s", issue, err)
glog.Error(err)
return nil, err
}
if issueInfo.Closed {
return nil, fmt.Errorf("Issue %d is closed.", issue)
}
}
git, err := gitinfo.CloneOrUpdate(repo, repoDir, false)
if err != nil {
glog.Fatalf("Failed to CloneOrUpdate repo %q: %s", repo, err)
}
if issue > 0 {
cmd := exec.Command("patch", "-p1")
if _, err := cmd.StdinPipe(); err != nil {
return nil, err
}
cmd.Dir = repoDir
diff, err := rc.Patchset(issue, patchset)
if err != nil {
return nil, fmt.Errorf("Failed to retrieve patchset: %s", err)
}
cmd.Stdin = diff
defer util.Close(diff)
b, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("Error while patching %#v - %s: %s", *cmd, string(b), err)
}
}
d := &DocSet{
repoDir: repoDir,
}
d.BuildNavigation()
if refresh {
go func() {
for _ = range time.Tick(config.REFRESH) {
util.LogErr(git.Update(true, false))
d.BuildNavigation()
}
}()
}
return d, nil
}
func main() {
defer common.LogPanic()
common.InitWithMetrics("android_stats", graphiteServer)
pollFreq, err := time.ParseDuration(*frequency)
if err != nil {
glog.Fatalf("Invalid value for frequency %q: %s", *frequency, err)
}
if err := generateStats(); err != nil {
glog.Fatal(err)
}
for _ = range time.Tick(pollFreq) {
if err := generateStats(); err != nil {
glog.Error(err)
}
}
}
// step does a single step in ingesting builds from tradefed and pushing the results into the buildbot database.
func step(targets []string, buildService *androidbuildinternal.Service) {
glog.Errorf("step: Begin")
if err := repos.Update(); err != nil {
glog.Errorf("Failed to update repos: %s", err)
return
}
// Loop over every target and look for skia commits in the builds.
for _, target := range targets {
r, err := buildService.Build.List().Branch(SKIA_BRANCH).BuildType("submitted").Target(target).ExtraFields("changeInfo").MaxResults(40).Do()
if err != nil {
glog.Errorf("Failed to load internal builds: %v", err)
continue
}
// Iterate over the builds in reverse order so we ingest the earlier Git
// hashes first and the more recent Git hashes later.
for i := len(r.Builds) - 1; i >= 0; i-- {
b := r.Builds[i]
commits := androidbuild.CommitsFromChanges(b.Changes)
glog.Infof("Commits: %#v", commits)
if len(commits) > 0 {
// Only look at the first commit in the list. The commits always appear in reverse chronological order, so
// the 0th entry is the most recent commit.
c := commits[0]
// Create a buildbot.Build from the build info.
key, build := buildFromCommit(b, c)
glog.Infof("Key: %s Hash: %s", key, c.Hash)
// If this was a failure then we need to check that there is a
// mirror failure on the main branch, at which point we will say
// that this is a warning.
if build.Results == buildbot.BUILDBOT_FAILURE && brokenOnMaster(buildService, target, b.BuildId) {
build.Results = buildbot.BUILDBOT_WARNING
}
if err := ingestBuild(build, c.Hash, target); err != nil {
glog.Error(err)
}
}
liveness.Update()
}
}
}
func NewPollingStatus(poll func() (interface{}, error), frequency time.Duration) (*PollingStatus, error) {
s := PollingStatus{
pollFn: poll,
stop: make(chan bool),
}
if err := s.poll(); err != nil {
return nil, err
}
go func(s *PollingStatus) {
ticker := time.Tick(frequency)
for {
select {
case <-s.stop:
return
case <-ticker:
if err := s.poll(); err != nil {
glog.Error(err)
}
}
}
}(&s)
return &s, nil
}
// SkpCommits returns the indices for all the commits that contain SKP updates.
func (g *GitInfo) SkpCommits(tile *tiling.Tile) ([]int, error) {
// Executes a git log command that looks like:
//
// git log --format=format:%H 32956400b4d8f33394e2cdef9b66e8369ba2a0f3..e7416bfc9858bde8fc6eb5f3bfc942bc3350953a SKP_VERSION
//
// The output should be a \n separated list of hashes that match.
first, last := tile.CommitRange()
cmd := exec.Command("git", "log", "--format=format:%H", first+".."+last, "SKP_VERSION")
cmd.Dir = g.dir
b, err := cmd.Output()
if err != nil {
glog.Error(string(b))
return nil, err
}
hashes := strings.Split(string(b), "\n")
ret := []int{}
for i, c := range tile.Commits {
if c.CommitTime != 0 && util.In(c.Hash, hashes) {
ret = append(ret, i)
}
}
return ret, nil
}
func main() {
common.Init()
webhook.MustInitRequestSaltFromFile(util.WebhookRequestSaltPath)
// Send start email.
emailsArr := util.ParseEmails(*emails)
emailsArr = append(emailsArr, util.CtAdmins...)
if len(emailsArr) == 0 {
glog.Error("At least one email address must be specified")
return
}
skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&frontend.ChromiumPerfUpdateVars{}, *gaeTaskID))
skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Chromium perf"))
// Ensure webapp is updated and email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
// Cleanup dirs after run completes.
defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.ChromiumPerfRunsDir))
defer skutil.RemoveAll(filepath.Join(util.StorageDir, util.BenchmarkRunsDir))
// Cleanup tmp files after the run.
defer util.CleanTmpDir()
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Running chromium perf task on workers")
defer glog.Flush()
if *pagesetType == "" {
glog.Error("Must specify --pageset_type")
return
}
if *benchmarkName == "" {
glog.Error("Must specify --benchmark_name")
return
}
if *runID == "" {
glog.Error("Must specify --run_id")
return
}
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Errorf("Could not instantiate gsutil object: %s", err)
return
}
remoteOutputDir := filepath.Join(util.ChromiumPerfRunsDir, *runID)
// Copy the patches to Google Storage.
skiaPatchName := *runID + ".skia.patch"
blinkPatchName := *runID + ".blink.patch"
chromiumPatchName := *runID + ".chromium.patch"
for _, patchName := range []string{skiaPatchName, blinkPatchName, chromiumPatchName} {
if err := gs.UploadFile(patchName, os.TempDir(), remoteOutputDir); err != nil {
glog.Errorf("Could not upload %s to %s: %s", patchName, remoteOutputDir, err)
return
}
}
skiaPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, skiaPatchName)
blinkPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, blinkPatchName)
chromiumPatchLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, remoteOutputDir, chromiumPatchName)
// Create the two required chromium builds (with patch and without the patch).
chromiumHash, skiaHash, err := util.CreateChromiumBuild(*runID, *targetPlatform, "", "", true)
if err != nil {
glog.Errorf("Could not create chromium build: %s", err)
return
}
// Reboot all workers to start from a clean slate.
util.RebootWorkers()
// Run the run_chromium_perf script on all workers.
runIDNoPatch := *runID + "-nopatch"
runIDWithPatch := *runID + "-withpatch"
chromiumBuildNoPatch := fmt.Sprintf("try-%s-%s-%s", chromiumHash, skiaHash, runIDNoPatch)
chromiumBuildWithPatch := fmt.Sprintf("try-%s-%s-%s", chromiumHash, skiaHash, runIDWithPatch)
runChromiumPerfCmdTemplate := "DISPLAY=:0 run_chromium_perf " +
"--worker_num={{.WorkerNum}} --log_dir={{.LogDir}} --pageset_type={{.PagesetType}} " +
"--chromium_build_nopatch={{.ChromiumBuildNoPatch}} --chromium_build_withpatch={{.ChromiumBuildWithPatch}} " +
"--run_id_nopatch={{.RunIDNoPatch}} --run_id_withpatch={{.RunIDWithPatch}} " +
"--benchmark_name={{.BenchmarkName}} --benchmark_extra_args=\"{{.BenchmarkExtraArgs}}\" " +
"--browser_extra_args_nopatch=\"{{.BrowserExtraArgsNoPatch}}\" --browser_extra_args_withpatch=\"{{.BrowserExtraArgsWithPatch}}\" " +
"--repeat_benchmark={{.RepeatBenchmark}} --target_platform={{.TargetPlatform}};"
runChromiumPerfTemplateParsed := template.Must(template.New("run_chromium_perf_cmd").Parse(runChromiumPerfCmdTemplate))
runChromiumPerfCmdBytes := new(bytes.Buffer)
if err := runChromiumPerfTemplateParsed.Execute(runChromiumPerfCmdBytes, struct {
WorkerNum string
LogDir string
PagesetType string
ChromiumBuildNoPatch string
ChromiumBuildWithPatch string
RunIDNoPatch string
RunIDWithPatch string
BenchmarkName string
BenchmarkExtraArgs string
BrowserExtraArgsNoPatch string
BrowserExtraArgsWithPatch string
RepeatBenchmark int
TargetPlatform string
}{
WorkerNum: util.WORKER_NUM_KEYWORD,
LogDir: util.GLogDir,
PagesetType: *pagesetType,
ChromiumBuildNoPatch: chromiumBuildNoPatch,
ChromiumBuildWithPatch: chromiumBuildWithPatch,
RunIDNoPatch: runIDNoPatch,
RunIDWithPatch: runIDWithPatch,
BenchmarkName: *benchmarkName,
BenchmarkExtraArgs: *benchmarkExtraArgs,
BrowserExtraArgsNoPatch: *browserExtraArgsNoPatch,
BrowserExtraArgsWithPatch: *browserExtraArgsWithPatch,
RepeatBenchmark: *repeatBenchmark,
TargetPlatform: *targetPlatform,
}); err != nil {
glog.Errorf("Failed to execute template: %s", err)
return
}
cmd := []string{
fmt.Sprintf("cd %s;", util.CtTreeDir),
"git pull;",
"make all;",
// The main command that runs run_chromium_perf on all workers.
runChromiumPerfCmdBytes.String(),
}
// Setting a 1 day timeout since it may take a while run benchmarks with many
// repeats.
if _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, 1*24*time.Hour); err != nil {
glog.Errorf("Error while running cmd %s: %s", cmd, err)
return
}
// If "--output-format=csv-pivot-table" was specified then merge all CSV files and upload.
if strings.Contains(*benchmarkExtraArgs, "--output-format=csv-pivot-table") {
for _, runID := range []string{runIDNoPatch, runIDWithPatch} {
if err := mergeUploadCSVFiles(runID, gs); err != nil {
glog.Errorf("Unable to merge and upload CSV files for %s: %s", runID, err)
}
}
}
// Compare the resultant CSV files using csv_comparer.py
noPatchCSVPath := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runIDNoPatch, runIDNoPatch+".output")
withPatchCSVPath := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, runIDWithPatch, runIDWithPatch+".output")
htmlOutputDir := filepath.Join(util.StorageDir, util.ChromiumPerfRunsDir, *runID, "html")
skutil.MkdirAll(htmlOutputDir, 0700)
htmlRemoteDir := filepath.Join(remoteOutputDir, "html")
htmlOutputLinkBase := util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, htmlRemoteDir) + "/"
htmlOutputLink = htmlOutputLinkBase + "index.html"
noPatchOutputLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, util.BenchmarkRunsDir, runIDNoPatch, "consolidated_outputs", runIDNoPatch+".output")
withPatchOutputLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, util.BenchmarkRunsDir, runIDWithPatch, "consolidated_outputs", runIDWithPatch+".output")
// Construct path to the csv_comparer python script.
_, currentFile, _, _ := runtime.Caller(0)
pathToPyFiles := filepath.Join(
filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
"py")
pathToCsvComparer := filepath.Join(pathToPyFiles, "csv_comparer.py")
args := []string{
pathToCsvComparer,
"--csv_file1=" + noPatchCSVPath,
"--csv_file2=" + withPatchCSVPath,
"--output_html=" + htmlOutputDir,
"--variance_threshold=" + strconv.FormatFloat(*varianceThreshold, 'f', 2, 64),
"--discard_outliers=" + strconv.FormatFloat(*discardOutliers, 'f', 2, 64),
"--absolute_url=" + htmlOutputLinkBase,
"--requester_email=" + *emails,
"--skia_patch_link=" + skiaPatchLink,
"--blink_patch_link=" + blinkPatchLink,
"--chromium_patch_link=" + chromiumPatchLink,
"--raw_csv_nopatch=" + noPatchOutputLink,
"--raw_csv_withpatch=" + withPatchOutputLink,
"--num_repeated=" + strconv.Itoa(*repeatBenchmark),
"--target_platform=" + *targetPlatform,
"--browser_args_nopatch=" + *browserExtraArgsNoPatch,
"--browser_args_withpatch=" + *browserExtraArgsWithPatch,
"--pageset_type=" + *pagesetType,
"--chromium_hash=" + chromiumHash,
"--skia_hash=" + skiaHash,
}
if err := util.ExecuteCmd("python", args, []string{}, 2*time.Hour, nil, nil); err != nil {
glog.Errorf("Error running csv_comparer.py: %s", err)
return
}
// Copy the HTML files to Google Storage.
if err := gs.UploadDir(htmlOutputDir, htmlRemoteDir, true); err != nil {
glog.Errorf("Could not upload %s to %s: %s", htmlOutputDir, htmlRemoteDir, err)
return
}
taskCompletedSuccessfully = true
}
// downloadRemoteDir downloads the specified Google Storage dir to the specified
// local dir. The local dir will be emptied and recreated. Handles multiple levels
// of directories.
func (gs *GsUtil) downloadRemoteDir(localDir, gsDir string) error {
// Empty the local dir.
util.RemoveAll(localDir)
// Create the local dir.
util.MkdirAll(localDir, 0700)
// The channel where the storage objects to be deleted will be sent to.
chStorageObjects := make(chan filePathToStorageObject, MAX_CHANNEL_SIZE)
req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/")
for req != nil {
resp, err := req.Do()
if err != nil {
return fmt.Errorf("Error occured while listing %s: %s", gsDir, err)
}
for _, result := range resp.Items {
fileName := filepath.Base(result.Name)
// If downloading from subdir then add it to the fileName.
fileGsDir := filepath.Dir(result.Name)
subDirs := strings.TrimPrefix(fileGsDir, gsDir)
if subDirs != "" {
dirTokens := strings.Split(subDirs, "/")
for i := range dirTokens {
fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
}
// Create the local directory.
util.MkdirAll(filepath.Join(localDir, filepath.Dir(fileName)), 0700)
}
chStorageObjects <- filePathToStorageObject{storageObject: result, filePath: fileName}
}
if len(resp.NextPageToken) > 0 {
req.PageToken(resp.NextPageToken)
} else {
req = nil
}
}
close(chStorageObjects)
// Kick off goroutines to download the storage objects.
var wg sync.WaitGroup
for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
wg.Add(1)
go func(goroutineNum int) {
defer wg.Done()
for obj := range chStorageObjects {
result := obj.storageObject
filePath := obj.filePath
respBody, err := getRespBody(result, gs.client)
if err != nil {
glog.Errorf("Could not fetch %s: %s", result.MediaLink, err)
return
}
defer util.Close(respBody)
outputFile := filepath.Join(localDir, filePath)
out, err := os.Create(outputFile)
if err != nil {
glog.Errorf("Unable to create file %s: %s", outputFile, err)
return
}
defer util.Close(out)
if _, err = io.Copy(out, respBody); err != nil {
glog.Error(err)
return
}
glog.Infof("Downloaded gs://%s/%s to %s with goroutine#%d", GS_BUCKET_NAME, result.Name, outputFile, goroutineNum)
// Sleep for a second after downloading file to avoid bombarding Cloud
// storage.
time.Sleep(time.Second)
}
}(i + 1)
}
wg.Wait()
return nil
}
func main() {
defer common.LogPanic()
worker_common.Init()
defer util.TimeTrack(time.Now(), "Capturing Archives")
defer glog.Flush()
// Create the task file so that the master knows this worker is still busy.
skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)
if *chromiumBuild == "" {
glog.Error("Must specify --chromium_build")
return
}
// Reset the local chromium checkout.
if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
return
}
// Sync the local chromium checkout.
if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
return
}
// Delete and remake the local webpage archives directory.
pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
skutil.RemoveAll(pathToArchives)
skutil.MkdirAll(pathToArchives, 0700)
// Instantiate GsUtil object.
gs, err := util.NewGsUtil(nil)
if err != nil {
glog.Error(err)
return
}
// Download the specified chromium build if it does not exist locally.
if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
glog.Error(err)
return
}
// Download pagesets if they do not exist locally.
if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
// Loop through all pagesets.
fileInfos, err := ioutil.ReadDir(pathToPagesets)
if err != nil {
glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
return
}
glog.Infof("The %s fileInfos are: %s", len(fileInfos), fileInfos)
for _, fileInfo := range fileInfos {
pagesetBaseName := filepath.Base(fileInfo.Name())
if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
// Ignore timestamp files and .pyc files.
continue
}
// Read the pageset.
pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())
decodedPageset, err := util.ReadPageset(pagesetPath)
if err != nil {
glog.Errorf("Could not read %s: %s", pagesetPath, err)
return
}
glog.Infof("===== Processing %s =====", pagesetPath)
args := []string{
util.CAPTURE_ARCHIVES_DEFAULT_CT_BENCHMARK,
"--extra-browser-args=--disable-setuid-sandbox",
"--browser=exact",
"--browser-executable=" + chromiumBinary,
"--user-agent=" + decodedPageset.UserAgent,
"--urls-list=" + decodedPageset.UrlsList,
"--archive-data-file=" + decodedPageset.ArchiveDataFile,
}
env := []string{
fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
"DISPLAY=:0",
}
skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
}
// Write timestamp to the webpage archives dir.
skutil.LogErr(util.CreateTimestampFile(pathToArchives))
// Upload webpage archives dir to Google Storage.
if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
glog.Error(err)
return
}
}