func digestFromIntermediate(test, digest string, inter *intermediate, e *expstorage.Expectations, tile *tiling.Tile, tallies *tally.Tallies, blamer *blame.Blamer, diffStore diff.DiffStore, paramset *paramsets.Summary, includeIgnores bool) *Digest { traceTally := tallies.ByTrace() ret := &Digest{ Test: test, Digest: digest, Status: e.Classification(test, digest).String(), Traces: buildTraces(test, digest, inter, e, tile, traceTally, paramset, includeIgnores), Diff: buildDiff(test, digest, inter, e, tile, tallies.ByTest(), blamer, diffStore, paramset, includeIgnores), } return ret }
// searchTile queries across a tile. func searchTile(q *Query, e *expstorage.Expectations, parsedQuery url.Values, storages *storage.Storage, tile *tiling.Tile, tallies *tally.Tallies, blamer *blame.Blamer, paramset *paramsets.Summary) ([]*Digest, []*tiling.Commit, error) { // TODO Use CommitRange to create a trimmed tile. traceTally := tallies.ByTrace() lastCommitIndex := tile.LastCommitIndex() // Loop over the tile and pull out all the digests that match // the query, collecting the matching traces as you go. Build // up a set of intermediate's that can then be used to calculate // Digest's. // map [test:digest] *intermediate inter := map[string]*intermediate{} for id, tr := range tile.Traces { if tiling.Matches(tr, parsedQuery) { test := tr.Params()[types.PRIMARY_KEY_FIELD] // Get all the digests digests := digestsFromTrace(id, tr, q.Head, lastCommitIndex, traceTally) for _, digest := range digests { cl := e.Classification(test, digest) if q.excludeClassification(cl) { continue } // Fix blamer to make this easier. if q.BlameGroupID != "" { if cl == types.UNTRIAGED { b := blamer.GetBlame(test, digest, tile.Commits) if q.BlameGroupID != blameGroupID(b, tile.Commits) { continue } } else { continue } } key := fmt.Sprintf("%s:%s", test, digest) if i, ok := inter[key]; !ok { inter[key] = newIntermediate(test, digest, id, tr, digests) } else { i.addTrace(id, tr, digests) } } } } // Now loop over all the intermediates and build a Digest for each one. ret := make([]*Digest, 0, len(inter)) for key, i := range inter { parts := strings.Split(key, ":") ret = append(ret, digestFromIntermediate(parts[0], parts[1], i, e, tile, tallies, blamer, storages.DiffStore, paramset, q.IncludeIgnores)) } return ret, tile.Commits, nil }
// searchByIssue searches across the given issue. func searchByIssue(issue string, q *Query, exp *expstorage.Expectations, parsedQuery url.Values, storages *storage.Storage, tile *tiling.Tile, tallies *tally.Tallies, tileParamSet *paramsets.Summary) ([]*Digest, error) { trybotResults, err := storages.TrybotResults.Get(issue) if err != nil { return nil, err } if !q.IncludeIgnores { matcher, err := storages.IgnoreStore.BuildRuleMatcher() if err != nil { return nil, fmt.Errorf("Unable to build rules matcher: %s", err) } for k, v := range trybotResults { if _, ok := matcher(v.Params); ok { delete(trybotResults, k) } } } rule := ignore.NewQueryRule(parsedQuery) // Aggregate the results into an intermediate representation to avoid // passing over the dataset twice. inter := map[string]*issueIntermediate{} for _, tbr := range trybotResults { if rule.IsMatch(tbr.Params) { key := tbr.Test + ":" + tbr.Digest if found, ok := inter[key]; ok { found.add(tbr) } else if cl := exp.Classification(tbr.Test, tbr.Digest); !q.excludeClassification(cl) { inter[key] = newIssueIntermediate(tbr, cl) } } } // Build the output. talliesByTest := tallies.ByTest() ret := make([]*Digest, 0, len(inter)) for _, i := range inter { ret = append(ret, &Digest{ Test: i.test, Digest: i.digest, Status: i.status.String(), ParamSet: i.paramSet, Diff: buildDiff(i.test, i.digest, exp, tile, talliesByTest, nil, storages.DiffStore, tileParamSet, q.IncludeIgnores), }) } return ret, nil }
// New creates a new Summary. // // The Summary listens for change events from the tallies object // and calculates new paramsets on every event. func New(tallies *tally.Tallies, storages *storage.Storage) *Summary { byTrace, byTraceIncludeIgnored, err := oneStep(tallies, storages) if err != nil { glog.Fatalf("Failed to calculate first step of paramsets: %s", err) } s := &Summary{ byTrace: byTrace, byTraceIncludeIgnored: byTraceIncludeIgnored, tallies: tallies, storages: storages, } tallies.OnChange(s.updateParamSets) return s }
// oneStep does a single step, calculating all the paramsets from the latest tile and tallies. // // Returns the paramsets for both the tile with and without ignored traces included. func oneStep(tallies *tally.Tallies, storages *storage.Storage) (map[string]map[string][]string, map[string]map[string][]string, error) { defer timer.New("paramsets").Stop() tile, err := storages.GetLastTileTrimmed(false) if err != nil { return nil, nil, fmt.Errorf("Failed to get tile: %s", err) } byTrace := byTraceForTile(tile, tallies.ByTrace()) tile, err = storages.GetLastTileTrimmed(true) if err != nil { return nil, nil, fmt.Errorf("Failed to get tile: %s", err) } byTraceIncludeIgnored := byTraceForTile(tile, tallies.ByTrace()) return byTrace, byTraceIncludeIgnored, nil }
// New creates a new instance of Summaries. func New(storages *storage.Storage, tallies *tally.Tallies, blamer *blame.Blamer) (*Summaries, error) { s := &Summaries{ storages: storages, tallies: tallies, blamer: blamer, } var err error s.summaries, err = s.CalcSummaries(nil, "", false, true) if err != nil { return nil, fmt.Errorf("Failed to calculate summaries in New: %s", err) } // TODO(jcgregorio) Move to a channel for tallies and then combine // this and the expStore handling into a single switch statement. tallies.OnChange(func() { summaries, err := s.CalcSummaries(nil, "", false, true) if err != nil { glog.Errorf("Failed to refresh summaries: %s", err) return } s.mutex.Lock() s.summaries = summaries s.mutex.Unlock() }) storages.EventBus.SubscribeAsync(expstorage.EV_EXPSTORAGE_CHANGED, func(e interface{}) { testNames := e.([]string) glog.Info("Updating summaries after expectations change.") partialSummaries, err := s.CalcSummaries(testNames, "", false, true) if err != nil { glog.Errorf("Failed to refresh summaries: %s", err) return } s.mutex.Lock() for k, v := range partialSummaries { s.summaries[k] = v } s.mutex.Unlock() }) return s, nil }
func Init(storages *storage.Storage, summaries *summary.Summaries, tallies *tally.Tallies) error { exp, err := storages.ExpectationsStore.Get() if err != nil { return err } go func() { oneRun := func() { t := timer.New("warmer one loop") for test, sum := range summaries.Get() { for _, digest := range sum.UntHashes { t := tallies.ByTest()[test] if t != nil { // Calculate the closest digest for the side effect of filling in the filediffstore cache. digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.POSITIVE) digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.NEGATIVE) } } } t.Stop() if newExp, err := storages.ExpectationsStore.Get(); err != nil { glog.Errorf("warmer: Failed to get expectations: %s", err) } else { exp = newExp } // Make sure all images are downloaded. This is necessary, because // the front-end doesn't get URLs (generated via DiffStore.AbsPath) // which ensures that the image has been downloaded. // TODO(stephana): Remove this once the new diffstore is in place. tile, err := storages.GetLastTileTrimmed(true) if err != nil { glog.Errorf("Error retrieving tile: %s", err) } tileLen := tile.LastCommitIndex() + 1 traceDigests := make(map[string]bool, tileLen) for _, trace := range tile.Traces { gTrace := trace.(*types.GoldenTrace) for _, digest := range gTrace.Values { if digest != types.MISSING_DIGEST { traceDigests[digest] = true } } } digests := util.KeysOfStringSet(traceDigests) glog.Infof("FOUND %d digests to fetch.", len(digests)) storages.DiffStore.AbsPath(digests) if err := warmTrybotDigests(storages, traceDigests); err != nil { glog.Errorf("Error retrieving trybot digests: %s", err) return } } oneRun() for _ = range time.Tick(time.Minute) { oneRun() } }() return nil }
// searchByIssue searches across the given issue. func searchByIssue(issue string, q *Query, exp *expstorage.Expectations, parsedQuery url.Values, storages *storage.Storage, tile *tiling.Tile, tallies *tally.Tallies, tileParamSet *paramsets.Summary) ([]*Digest, error) { trybotResults, err := storages.TrybotResults.Get(issue) if err != nil { return nil, err } // Get a matcher for the ignore rules if we filter ignores. var ignoreMatcher ignore.RuleMatcher = nil if !q.IncludeIgnores { ignoreMatcher, err = storages.IgnoreStore.BuildRuleMatcher() if err != nil { return nil, fmt.Errorf("Unable to build rules matcher: %s", err) } } // Set up a rule to match the query. var queryRule ignore.QueryRule = nil if len(parsedQuery) > 0 { queryRule = ignore.NewQueryRule(parsedQuery) } // Aggregate the results into an intermediate representation to avoid // passing over the dataset twice. inter := map[string]*issueIntermediate{} talliesByTest := tallies.ByTest() for _, bot := range trybotResults.Bots { for _, result := range bot.TestResults { expandedParams := util.CopyStringMap(bot.BotParams) util.AddParams(expandedParams, result.Params) if ignoreMatcher != nil { if _, ok := ignoreMatcher(expandedParams); ok { continue } } if (queryRule == nil) || queryRule.IsMatch(expandedParams) { testName := expandedParams[types.PRIMARY_KEY_FIELD] digest := trybotResults.Digests[result.DigestIdx] key := testName + ":" + digest if !q.IncludeMaster { if _, ok := talliesByTest[testName][digest]; ok { continue } } if found, ok := inter[key]; ok { found.add(expandedParams) } else if cl := exp.Classification(testName, digest); !q.excludeClassification(cl) { inter[key] = newIssueIntermediate(expandedParams, digest, cl) } } } } // Build the output and make sure the digest are cached on disk. digests := make(map[string]bool, len(inter)) ret := make([]*Digest, 0, len(inter)) emptyTraces := &Traces{} for _, i := range inter { ret = append(ret, &Digest{ Test: i.test, Digest: i.digest, Status: i.status.String(), ParamSet: i.paramSet, Diff: buildDiff(i.test, i.digest, exp, tile, talliesByTest, nil, storages.DiffStore, tileParamSet, q.IncludeIgnores), Traces: emptyTraces, }) digests[i.digest] = true } // This ensures that all digests are cached on disk. storages.DiffStore.AbsPath(util.KeysOfStringSet(digests)) return ret, nil }
// Search returns a slice of Digests that match the input query, and the total number of Digests // that matched the query. It also returns a slice of Commits that were used in the calculations. func Search(q *Query, storages *storage.Storage, tallies *tally.Tallies, blamer *blame.Blamer, paramset *paramsets.Summary) ([]*Digest, int, []*tiling.Commit, error) { tile, err := storages.GetLastTileTrimmed(q.IncludeIgnores) if err != nil { return nil, 0, nil, fmt.Errorf("Couldn't retrieve tile: %s", err) } // TODO Use CommitRange to create a trimmed tile. parsedQuery, err := url.ParseQuery(q.Query) if err != nil { return nil, 0, nil, fmt.Errorf("Failed to parse Query in Search: %s", err) } e, err := storages.ExpectationsStore.Get() if err != nil { return nil, 0, nil, fmt.Errorf("Couldn't get expectations: %s", err) } traceTally := tallies.ByTrace() lastCommitIndex := tile.LastCommitIndex() // Loop over the tile and pull out all the digests that match // the query, collecting the matching traces as you go. Build // up a set of intermediate's that can then be used to calculate // Digest's. // map [test:digest] *intermediate inter := map[string]*intermediate{} for id, tr := range tile.Traces { if tiling.Matches(tr, parsedQuery) { test := tr.Params()[types.PRIMARY_KEY_FIELD] // Get all the digests digests := digestsFromTrace(id, tr, q.Head, lastCommitIndex, traceTally) for _, digest := range digests { cl := e.Classification(test, digest) switch { case cl == types.NEGATIVE && !q.Neg: continue case cl == types.POSITIVE && !q.Pos: continue case cl == types.UNTRIAGED && !q.Unt: continue } // Fix blamer to make this easier. if q.BlameGroupID != "" { if cl == types.UNTRIAGED { b := blamer.GetBlame(test, digest, tile.Commits) if q.BlameGroupID != blameGroupID(b, tile.Commits) { continue } } else { continue } } key := fmt.Sprintf("%s:%s", test, digest) if i, ok := inter[key]; !ok { inter[key] = newIntermediate(test, digest, id, tr, digests) } else { i.addTrace(id, tr, digests) } } } } // Now loop over all the intermediates and build a Digest for each one. ret := make([]*Digest, 0, len(inter)) for key, i := range inter { parts := strings.Split(key, ":") ret = append(ret, digestFromIntermediate(parts[0], parts[1], i, e, tile, tallies, blamer, storages.DiffStore, paramset, q.IncludeIgnores)) } sort.Sort(DigestSlice(ret)) fullLength := len(ret) if fullLength > q.Limit { ret = ret[0:q.Limit] } return ret, fullLength, tile.Commits, nil }