func sendEmail(recipients []string) { // Send completion email. emailSubject := fmt.Sprintf("Run lua script Cluster telemetry task has completed (%s)", *runID) failureHtml := "" if !taskCompletedSuccessfully { emailSubject += " with failures" failureHtml = util.GetFailureEmailHtml(*runID) } scriptOutputHtml := "" if luaOutputRemoteLink != "" { scriptOutputHtml = fmt.Sprintf("The output of your script is available <a href='%s'>here</a>.<br/>\n", luaOutputRemoteLink) } aggregatorOutputHtml := "" if luaAggregatorOutputRemoteLink != "" { aggregatorOutputHtml = fmt.Sprintf("The aggregated output of your script is available <a href='%s'>here</a>.<br/>\n", luaAggregatorOutputRemoteLink) } bodyTemplate := ` The Cluster telemetry queued task to run lua script on %s pageset has completed.<br/> Run description: %s<br/> %s %s %s You can schedule more runs <a href="%s">here</a>.<br/><br/> Thanks! ` emailBody := fmt.Sprintf(bodyTemplate, *pagesetType, *description, failureHtml, scriptOutputHtml, aggregatorOutputHtml, frontend.LuaTasksWebapp) if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } }
func sendEmail(recipients []string) { // Send completion email. emailSubject := fmt.Sprintf("Cluster telemetry chromium perf task has completed (%s)", *runID) failureHtml := "" if !taskCompletedSuccessfully { emailSubject += " with failures" failureHtml = util.FailureEmailHtml } bodyTemplate := ` The chromium perf %s benchmark task on %s pageset has completed.<br/> %s The HTML output with differences between the base run and the patch run is <a href='%s'>here</a>.<br/> The patch(es) you specified are here: <a href='%s'>chromium</a>/<a href='%s'>blink</a>/<a href='%s'>skia</a> <br/><br/> You can schedule more runs <a href='%s'>here</a>. <br/><br/> Thanks! ` emailBody := fmt.Sprintf(bodyTemplate, *benchmarkName, *pagesetType, failureHtml, htmlOutputLink, chromiumPatchLink, blinkPatchLink, skiaPatchLink, frontend.ChromiumPerfTasksWebapp) if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } }
func sendEmail(recipients []string) { emailSubject := "Chromium build task has completed" failureHtml := "" if !taskCompletedSuccessfully { emailSubject += " with failures" failureHtml = util.GetFailureEmailHtml(*runID) } bodyTemplate := ` The Cluster telemetry queued task to create a new chromium build has completed.<br/> %s You can schedule more runs <a href="%s">here</a>.<br/><br/> Thanks! ` emailBody := fmt.Sprintf(bodyTemplate, failureHtml, frontend.ChromiumBuildTasksWebapp) if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } }
func sendEmail(recipients []string) { // Send completion email. emailSubject := "Create pagesets Cluster telemetry task has completed" failureHtml := "" if !*taskCompletedSuccessfully { emailSubject += " with failures" failureHtml = util.FailureEmailHtml } bodyTemplate := ` The Cluster telemetry queued task to create %s pagesets has completed.<br/> %s You can schedule more runs <a href="%s">here</a>.<br/><br/> Thanks! ` emailBody := fmt.Sprintf(bodyTemplate, *pagesetType, failureHtml, frontend.AdminTasksWebapp) if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } }
func sendEmail(recipients []string) { // Send completion email. emailSubject := fmt.Sprintf("Cluster telemetry fix archives task has completed (%s)", *runID) failureHtml := "" if !taskCompletedSuccessfully { emailSubject += " with failures" failureHtml = util.GetFailureEmailHtml(*runID) } bodyTemplate := ` The Cluster telemetry fix archives task on %s pageset has completed.<br/> %s The output of your script is available <a href='%s'>here</a>.<br/><br/> Thanks! ` emailBody := fmt.Sprintf(bodyTemplate, *pagesetType, failureHtml, outputRemoteLink) if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } }
func sendEmail(recipients []string) { // Send completion email. emailSubject := fmt.Sprintf("Capture SKPs cluster telemetry task has completed (%s)", *runID) failureHtml := "" if !taskCompletedSuccessfully { emailSubject += " with failures" failureHtml = util.FailureEmailHtml } bodyTemplate := ` The Capture SKPs task on %s pageset has completed.<br/> %s The output of your script is available <a href='%s'>here</a>.<br/><br/> You can schedule more runs <a href="%s">here</a>.<br/><br/> Thanks! ` emailBody := fmt.Sprintf(bodyTemplate, *pagesetType, failureHtml, outputRemoteLink, frontend.CaptureSKPsTasksWebapp) if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } }
func main() { defer common.LogPanic() common.Init() defer glog.Flush() // Collect unhealthy machines offlineMachines := []string{} offlineDevices := []string{} missingDevices := []string{} nonResponsiveDevices := []string{} // Also collect healthy machines for additional checks. healthyMachines := []string{} deviceOfflineOutputs, err := util.SSH("adb devices", util.Slaves, util.ADB_DEVICES_TIMEOUT) if err != nil { glog.Fatalf("Error while sshing into workers: %s", err) return } // Populate offlineMachines, offlineDevices and missingDevices. for hostname, out := range deviceOfflineOutputs { if out == "" { offlineMachines = append(offlineMachines, hostname) glog.Warningf("%s is offline", hostname) } else if strings.Contains(out, "offline") { // The adb output text contains offline devices. offlineDevices = append(offlineDevices, hostname) glog.Warningf("%s has an offline device", hostname) } else if strings.Count(out, "device") == 1 { // The adb output text only contains "List of devices attached" // without any devices listed below it. missingDevices = append(missingDevices, hostname) glog.Warningf("%s has missing devices", hostname) } else { // Everything seems fine so far, add this machine as a healthyMachine. healthyMachines = append(healthyMachines, hostname) } } // Populate nonResponsiveDevices. responsivenessOutputs, err := util.SSH("adb shell uptime", healthyMachines, util.ADB_SHELL_UPTIME_TIMEOUT) if err != nil { glog.Fatalf("Error while sshing into workers: %s", err) return } // Clear and repopulate the healthy machines slice. healthyMachines = nil for hostname, out := range responsivenessOutputs { if out == "" { nonResponsiveDevices = append(nonResponsiveDevices, hostname) glog.Warningf("%s has non-responsive devices.") } else { // Everything seems fine so far, add this machine as a healthyMachine. healthyMachines = append(healthyMachines, hostname) } } // Email admins if there are any unhealthy machines. if len(offlineMachines) != 0 || len(offlineDevices) != 0 || len(missingDevices) != 0 || len(nonResponsiveDevices) != 0 { emailSubject := "There are unhealthy Cluster telemetry machines" emailBody := "Please file a ticket to chrome-golo-tech-ticket@ (for offline devices) and chrome-labs-tech-ticket@ (for offline machines) using https://docs.google.com/spreadsheets/d/1whlE4nDJB0XFBemJliupOORepdXf_vXyAfFgsprTAxY/edit#gid=0 for-<br/><br/>" if len(offlineMachines) != 0 { emailBody += fmt.Sprintf("The following machines are offline: %s<br/>", strings.Join(offlineMachines, ",")) } if len(offlineDevices) != 0 { emailBody += fmt.Sprintf("The following machines have offline devices: %s<br/>", strings.Join(offlineDevices, ",")) } if len(missingDevices) != 0 { emailBody += fmt.Sprintf("The following machines have missing devices: %s<br/>", strings.Join(missingDevices, ",")) } if len(nonResponsiveDevices) != 0 { emailBody += fmt.Sprintf("The following machines have non-responsive devices: %s<br/>", strings.Join(nonResponsiveDevices, ",")) } if err := util.SendEmail(util.CtAdmins, emailSubject, emailBody); err != nil { glog.Errorf("Error while sending email: %s", err) return } } else { glog.Info("All CT machines are healthy") } }