// clusteringHandler handles doing the actual k-means clustering. // // The return format is JSON of the form: // // { // "Clusters": [ // { // "Keys": [ // "x86:GeForce320M:MacMini4.1:Mac10.8:GM_varied_text_clipped_no_lcd_640_480:8888",...], // "ParamSummaries": [ // [{"Value": "Win8", "Weight": 15}, {"Value": "Android", "Weight": 14}, ...] // ], // "StepFit": { // "LeastSquares":0.0006582442047814354, // "TurningPoint":162, // "StepSize":0.023272272692293046, // "Regression": 35.3 // } // Traces: [[[0, -0.00007967326606768456], [1, 0.011877665949459049], [2, 0.012158129176717419],...]] // }, // ... // ], // "K": 5, // "StdDevThreshhold": 0.1 // } // // Note that Keys contains all the keys, while Traces only contains traces of // the N closest cluster members and the centroid. // // Takes the following query parameters: // // _k - The K to use for k-means clustering. // _stddev - The standard deviation to use when normalize traces // during k-means clustering. // _issue - The Rietveld issue ID with trybot results to include. // // Additionally the rest of the query parameters as returned from // sk.Query.selectionsAsQuery(). func clusteringHandler(w http.ResponseWriter, r *http.Request) { glog.Infof("Clustering Handler: %q\n", r.URL.Path) tile := masterTileBuilder.GetTile() w.Header().Set("Content-Type", "application/json") // If there are no query parameters just return with an empty set of ClusterSummaries. if r.FormValue("_k") == "" || r.FormValue("_stddev") == "" { writeClusterSummaries(clustering.NewClusterSummaries(), w, r) return } k, err := strconv.ParseInt(r.FormValue("_k"), 10, 32) if err != nil { util.ReportError(w, r, err, fmt.Sprintf("_k parameter must be an integer %s.", r.FormValue("_k"))) return } stddev, err := strconv.ParseFloat(r.FormValue("_stddev"), 64) if err != nil { util.ReportError(w, r, err, fmt.Sprintf("_stddev parameter must be a float %s.", r.FormValue("_stddev"))) return } issue := r.FormValue("_issue") var tryResults *types.TryBotResults = nil if issue != "" { var err error tryResults, err = trybot.Get(issue) if err != nil { util.ReportError(w, r, err, fmt.Sprintf("Failed to get trybot data for clustering.")) return } } delete(r.Form, "_k") delete(r.Form, "_stddev") delete(r.Form, "_issue") // Create a filter function for traces that match the query parameters and // optionally tryResults. filter := func(key string, tr *types.PerfTrace) bool { if tryResults != nil { if _, ok := tryResults.Values[key]; !ok { return false } } return tiling.Matches(tr, r.Form) } if issue != "" { if tile, err = trybot.TileWithTryData(tile, issue); err != nil { util.ReportError(w, r, err, fmt.Sprintf("Failed to get trybot data for clustering.")) return } } summary, err := clustering.CalculateClusterSummaries(tile, int(k), stddev, filter) if err != nil { util.ReportError(w, r, err, "Failed to calculate clusters.") return } writeClusterSummaries(summary, w, r) }
// singleStep does a single round of alerting. func singleStep(issueTracker issues.IssueTracker) { latencyBegin := time.Now() tile := tileBuilder.GetTile() summary, err := clustering.CalculateClusterSummaries(tile, CLUSTER_SIZE, CLUSTER_STDDEV, skpOnly) if err != nil { glog.Errorf("Alerting: Failed to calculate clusters: %s", err) return } fresh := []*types.ClusterSummary{} for _, c := range summary.Clusters { if math.Abs(c.StepFit.Regression) > clustering.INTERESTING_THRESHHOLD { fresh = append(fresh, c) } } old, err := ListFrom(tile.Commits[0].CommitTime) if err != nil { glog.Errorf("Alerting: Failed to get existing clusters: %s", err) return } glog.Infof("Found %d old", len(old)) glog.Infof("Found %d fresh", len(fresh)) updated := CombineClusters(fresh, old) glog.Infof("Found %d to update", len(updated)) for _, c := range updated { if c.Status == "" { c.Status = "New" } if err := Write(c); err != nil { glog.Errorf("Alerting: Failed to write updated cluster: %s", err) } } current, err := ListFrom(tile.Commits[0].CommitTime) if err != nil { glog.Errorf("Alerting: Failed to get existing clusters: %s", err) return } count := 0 for _, c := range current { if c.Status == "New" { count++ } if issueTracker != nil { if err := updateBugs(c, issueTracker); err != nil { glog.Errorf("Error retrieving bugs: %s", err) return } } else { glog.Infof("Skipping ClusterSummary.Bugs update because apiKey is missing.") return } } newClustersGauge.Update(int64(count)) runsCounter.Inc(1) alertingLatency.UpdateSince(latencyBegin) }