// buildDiff creates a Diff for the given intermediate.
func buildDiff(test, digest string, e *expstorage.Expectations, tile *tiling.Tile, testTally map[string]tally.Tally, blamer *blame.Blamer, diffStore diff.DiffStore, paramset *paramsets.Summary, includeIgnores bool) *Diff {
ret := &Diff{
Diff: math.MaxFloat32,
Pos: nil,
Neg: nil,
}
if blamer != nil {
ret.Blame = blamer.GetBlame(test, digest, tile.Commits)
}
t := testTally[test]
if t == nil {
t = tally.Tally{}
}
ret.Pos = &DiffDigest{
Closest: digesttools.ClosestDigest(test, digest, e, t, diffStore, types.POSITIVE),
}
ret.Pos.ParamSet = paramset.Get(test, ret.Pos.Closest.Digest, includeIgnores)
ret.Neg = &DiffDigest{
Closest: digesttools.ClosestDigest(test, digest, e, t, diffStore, types.NEGATIVE),
}
ret.Neg.ParamSet = paramset.Get(test, ret.Neg.Closest.Digest, includeIgnores)
if pos, neg := ret.Pos.Closest.Diff, ret.Neg.Closest.Diff; pos < neg {
ret.Diff = pos
} else {
ret.Diff = neg
}
return ret
}
func buildDetailsGUI(tile *tiling.Tile, exp *expstorage.Expectations, test string, top string, left string, graphs bool, closest bool, includeIgnores bool) *PolyDetailsGUI {
ret := &PolyDetailsGUI{
TopStatus: exp.Classification(test, top).String(),
LeftStatus: exp.Classification(test, left).String(),
Params: []*PerParamCompare{},
Traces: []*Trace{},
TileSize: len(tile.Commits),
}
topParamSet := paramsetSum.Get(test, top, includeIgnores)
leftParamSet := paramsetSum.Get(test, left, includeIgnores)
traceNames := []string{}
tally := tallies.ByTrace()
for id, tr := range tile.Traces {
if tr.Params()[types.PRIMARY_KEY_FIELD] == test {
traceNames = append(traceNames, id)
}
}
keys := util.UnionStrings(util.KeysOfParamSet(topParamSet), util.KeysOfParamSet(leftParamSet))
sort.Strings(keys)
for _, k := range keys {
ret.Params = append(ret.Params, &PerParamCompare{
Name: k,
Top: safeGet(topParamSet, k),
Left: safeGet(leftParamSet, k),
})
}
// Now build the trace data.
if graphs {
ret.Traces, ret.OtherDigests = buildTraceData(top, traceNames, tile, tally, exp)
ret.Commits = tile.Commits
ret.Blame = blamer.GetBlame(test, top, ret.Commits)
}
// Now find the closest positive and negative digests.
t := tallies.ByTest()[test]
if closest && t != nil {
ret.PosClosest = digesttools.ClosestDigest(test, top, exp, t, storages.DiffStore, types.POSITIVE)
ret.NegClosest = digesttools.ClosestDigest(test, top, exp, t, storages.DiffStore, types.NEGATIVE)
}
if top == left {
var err error
// Search is only done on the digest. Codesite can't seem to extract the
// name of the test reliably from the URL in comment text, yet can get the
// digest just fine. This issue should be revisited once we switch to
// Monorail.
ret.Issues, err = issueTracker.FromQuery(top)
if err != nil {
glog.Errorf("Failed to load issues for [%s, %s]: %s", test, top, err)
}
}
return ret
}
// buildDiff creates a Diff for the given intermediate.
func buildDiff(test, digest string, e *expstorage.Expectations, tile *tiling.Tile, testTally map[string]tally.Tally, blamer *blame.Blamer, diffStore diff.DiffStore, paramset *paramsets.Summary, includeIgnores bool) *Diff {
ret := &Diff{
Diff: math.MaxFloat32,
Pos: nil,
Neg: nil,
}
if blamer != nil {
ret.Blame = blamer.GetBlame(test, digest, tile.Commits)
}
t := testTally[test]
if t == nil {
t = tally.Tally{}
}
var diffVal float32 = 0
if closest := digesttools.ClosestDigest(test, digest, e, t, diffStore, types.POSITIVE); closest.Digest != "" {
ret.Pos = &DiffDigest{
Closest: closest,
}
ret.Pos.ParamSet = paramset.Get(test, ret.Pos.Closest.Digest, includeIgnores)
diffVal = closest.Diff
}
if closest := digesttools.ClosestDigest(test, digest, e, t, diffStore, types.NEGATIVE); closest.Digest != "" {
ret.Neg = &DiffDigest{
Closest: closest,
}
ret.Neg.ParamSet = paramset.Get(test, ret.Neg.Closest.Digest, includeIgnores)
if (ret.Pos == nil) || (closest.Diff < diffVal) {
diffVal = closest.Diff
}
}
ret.Diff = diffVal
return ret
}
func Init(storages *storage.Storage, summaries *summary.Summaries, tallies *tally.Tallies) error {
exp, err := storages.ExpectationsStore.Get()
if err != nil {
return err
}
go func() {
oneRun := func() {
t := timer.New("warmer one loop")
for test, sum := range summaries.Get() {
for _, digest := range sum.UntHashes {
t := tallies.ByTest()[test]
if t != nil {
// Calculate the closest digest for the side effect of filling in the filediffstore cache.
digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.POSITIVE)
digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.NEGATIVE)
}
}
}
t.Stop()
if newExp, err := storages.ExpectationsStore.Get(); err != nil {
glog.Errorf("warmer: Failed to get expectations: %s", err)
} else {
exp = newExp
}
// Make sure all images are downloaded. This is necessary, because
// the front-end doesn't get URLs (generated via DiffStore.AbsPath)
// which ensures that the image has been downloaded.
// TODO(stephana): Remove this once the new diffstore is in place.
tile, err := storages.GetLastTileTrimmed(true)
if err != nil {
glog.Errorf("Error retrieving tile: %s", err)
}
tileLen := tile.LastCommitIndex() + 1
traceDigests := make(map[string]bool, tileLen)
for _, trace := range tile.Traces {
gTrace := trace.(*types.GoldenTrace)
for _, digest := range gTrace.Values {
if digest != types.MISSING_DIGEST {
traceDigests[digest] = true
}
}
}
digests := util.KeysOfStringSet(traceDigests)
glog.Infof("FOUND %d digests to fetch.", len(digests))
storages.DiffStore.AbsPath(digests)
if err := warmTrybotDigests(storages, traceDigests); err != nil {
glog.Errorf("Error retrieving trybot digests: %s", err)
return
}
}
oneRun()
for _ = range time.Tick(time.Minute) {
oneRun()
}
}()
return nil
}