/**
Conditions to test.
Traces
------
id | config | test name | corpus(source_type) | digests
a 8888 foo gm aaa+, bbb-
b 565 foo gm ccc?, ddd?
c gpu foo gm eee+
d 8888 bar gm fff-, ggg?
e 8888 quux image jjj?
Expectations
------------
foo aaa pos
foo bbb neg
foo ccc unt
foo ddd unt
foo eee pos
bar fff neg
Ignores
-------
config=565
Note no entry for quux or ggg, meaning untriaged.
Test the following conditions and make sure you get
the expected test summaries.
source_type=gm
foo - pos(aaa, eee):2 neg(bbb):1
bar - neg(fff):1 unt(ggg):1
source_type=gm includeIgnores=true
foo - pos(aaa, eee):2 neg(bbb):1 unt(ccc, ddd):2
bar - neg(fff):1 unt(ggg):1
source_type=gm includeIgnores=true testName=foo
foo - pos(aaa, eee):2 neg(bbb):1 unt(ccc, ddd):2
testname = foo
foo - pos(aaa, eee):2 neg(bbb):1
testname = quux
quux - unt(jjj):1
config=565&config=8888
foo - pos(aaa):1 neg(bbb):1
bar - neg(fff):1 unt(ggg):1
quux - unt(jjj):1
config=565&config=8888 head=true
foo - neg(bbb):1
bar - unt(ggg):1
quux - unt(jjj):1
config=gpu
foo - pos(eee):1
config=unknown
<empty>
*/
func TestCalcSummaries(t *testing.T) {
tile := &tiling.Tile{
Traces: map[string]tiling.Trace{
"a": &types.GoldenTrace{
Values: []string{"aaa", "bbb"},
Params_: map[string]string{
"name": "foo",
"config": "8888",
"source_type": "gm"},
},
"b": &types.GoldenTrace{
Values: []string{"ccc", "ddd"},
Params_: map[string]string{
"name": "foo",
"config": "565",
"source_type": "gm"},
},
"c": &types.GoldenTrace{
Values: []string{"eee", types.MISSING_DIGEST},
Params_: map[string]string{
"name": "foo",
"config": "gpu",
"source_type": "gm"},
},
"d": &types.GoldenTrace{
Values: []string{"fff", "ggg"},
Params_: map[string]string{
"name": "bar",
"config": "8888",
"source_type": "gm"},
},
"e": &types.GoldenTrace{
Values: []string{"jjj", types.MISSING_DIGEST},
Params_: map[string]string{
"name": "quux",
"config": "8888",
"source_type": "image"},
},
},
Commits: []*tiling.Commit{
&tiling.Commit{
CommitTime: 42,
Hash: "ffffffffffffffffffffffffffffffffffffffff",
Author: "*****@*****.**",
},
&tiling.Commit{
CommitTime: 45,
Hash: "gggggggggggggggggggggggggggggggggggggggg",
Author: "*****@*****.**",
},
},
Scale: 0,
TileIndex: 0,
}
eventBus := eventbus.New(nil)
storages := &storage.Storage{
DiffStore: mocks.MockDiffStore{},
ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
IgnoreStore: ignore.NewMemIgnoreStore(),
MasterTileBuilder: mocks.NewMockTileBuilderFromTile(t, tile),
NCommits: 50,
EventBus: eventBus,
DigestStore: &mocks.MockDigestStore{FirstSeen: time.Now().Unix() + 1000, OkValue: true},
}
assert.Nil(t, storages.ExpectationsStore.AddChange(map[string]types.TestClassification{
"foo": map[string]types.Label{
"aaa": types.POSITIVE,
"bbb": types.NEGATIVE,
"ccc": types.UNTRIAGED,
"ddd": types.UNTRIAGED,
"eee": types.POSITIVE,
},
"bar": map[string]types.Label{
"fff": types.NEGATIVE,
},
}, "*****@*****.**"))
ta, _ := tally.New(storages)
assert.Nil(t, storages.IgnoreStore.Create(&ignore.IgnoreRule{
ID: 1,
Name: "foo",
Expires: time.Now().Add(time.Hour),
Query: "config=565",
}))
blamer, err := blame.New(storages)
assert.Nil(t, err)
summaries, err := New(storages, ta, blamer)
assert.Nil(t, err)
sum, err := summaries.CalcSummaries(nil, "source_type=gm", false, false)
if err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 2, len(sum))
triageCountsCorrect(t, sum, "foo", 2, 1, 0)
triageCountsCorrect(t, sum, "bar", 0, 1, 1)
assert.Equal(t, []string{}, sum["foo"].UntHashes)
assert.Equal(t, []string{"ggg"}, sum["bar"].UntHashes)
if sum, err = summaries.CalcSummaries(nil, "source_type=gm", true, false); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 2, len(sum))
triageCountsCorrect(t, sum, "foo", 2, 1, 2)
triageCountsCorrect(t, sum, "bar", 0, 1, 1)
assert.Equal(t, sum["foo"].UntHashes, []string{"ccc", "ddd"})
assert.Equal(t, sum["bar"].UntHashes, []string{"ggg"})
if sum, err = summaries.CalcSummaries([]string{"foo"}, "source_type=gm", true, false); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 1, len(sum))
triageCountsCorrect(t, sum, "foo", 2, 1, 2)
assert.Equal(t, sum["foo"].UntHashes, []string{"ccc", "ddd"})
if sum, err = summaries.CalcSummaries([]string{"foo"}, "", false, false); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 1, len(sum))
triageCountsCorrect(t, sum, "foo", 2, 1, 0)
assert.Equal(t, sum["foo"].UntHashes, []string{})
if sum, err = summaries.CalcSummaries(nil, "config=8888&config=565", false, false); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 3, len(sum))
triageCountsCorrect(t, sum, "foo", 1, 1, 0)
triageCountsCorrect(t, sum, "bar", 0, 1, 1)
triageCountsCorrect(t, sum, "quux", 0, 0, 1)
assert.Equal(t, sum["foo"].UntHashes, []string{})
assert.Equal(t, sum["bar"].UntHashes, []string{"ggg"})
assert.Equal(t, sum["quux"].UntHashes, []string{"jjj"})
if sum, err = summaries.CalcSummaries(nil, "config=8888&config=565", false, true); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 3, len(sum))
triageCountsCorrect(t, sum, "foo", 0, 1, 0)
triageCountsCorrect(t, sum, "bar", 0, 0, 1)
triageCountsCorrect(t, sum, "quux", 0, 0, 1)
assert.Equal(t, sum["foo"].UntHashes, []string{})
assert.Equal(t, sum["bar"].UntHashes, []string{"ggg"})
assert.Equal(t, sum["quux"].UntHashes, []string{"jjj"})
if sum, err = summaries.CalcSummaries(nil, "config=gpu", false, false); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 1, len(sum))
triageCountsCorrect(t, sum, "foo", 1, 0, 0)
assert.Equal(t, sum["foo"].UntHashes, []string{})
if sum, err = summaries.CalcSummaries(nil, "config=unknown", false, false); err != nil {
t.Fatalf("Failed to calc: %s", err)
}
assert.Equal(t, 0, len(sum))
}
func TestBlamerWithSyntheticData(t *testing.T) {
start := time.Now().Unix()
commits := []*tiling.Commit{
&tiling.Commit{CommitTime: start + 10, Hash: "h1", Author: "John Doe 1"},
&tiling.Commit{CommitTime: start + 20, Hash: "h2", Author: "John Doe 2"},
&tiling.Commit{CommitTime: start + 30, Hash: "h3", Author: "John Doe 3"},
&tiling.Commit{CommitTime: start + 40, Hash: "h4", Author: "John Doe 4"},
&tiling.Commit{CommitTime: start + 50, Hash: "h5", Author: "John Doe 5"},
}
params := []map[string]string{
map[string]string{"name": "foo", "config": "8888", "source_type": "gm"},
map[string]string{"name": "foo", "config": "565", "source_type": "gm"},
map[string]string{"name": "foo", "config": "gpu", "source_type": "gm"},
map[string]string{"name": "bar", "config": "8888", "source_type": "gm"},
map[string]string{"name": "bar", "config": "565", "source_type": "gm"},
map[string]string{"name": "bar", "config": "gpu", "source_type": "gm"},
map[string]string{"name": "baz", "config": "565", "source_type": "gm"},
map[string]string{"name": "baz", "config": "gpu", "source_type": "gm"},
}
DI_1, DI_2, DI_3 := "digest1", "digest2", "digest3"
DI_4, DI_5, DI_6 := "digest4", "digest5", "digest6"
DI_7, DI_8, DI_9 := "digest7", "digest8", "digest9"
MISS := types.MISSING_DIGEST
digests := [][]string{
[]string{MISS, MISS, DI_1, MISS, MISS},
[]string{MISS, DI_1, DI_1, DI_2, MISS},
[]string{DI_3, MISS, MISS, MISS, MISS},
[]string{DI_5, DI_4, DI_5, DI_5, DI_5},
[]string{DI_6, MISS, DI_4, MISS, MISS},
[]string{MISS, MISS, MISS, MISS, MISS},
[]string{DI_7, DI_7, MISS, DI_8, MISS},
[]string{DI_7, MISS, DI_7, DI_8, MISS},
}
// Make sure the data are consistent and create a mock TileStore.
assert.Equal(t, len(commits), len(digests[0]))
assert.Equal(t, len(digests), len(params))
eventBus := eventbus.New(nil)
storages := &storage.Storage{
ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
MasterTileBuilder: mocks.NewMockTileBuilder(t, digests, params, commits),
DigestStore: &mocks.MockDigestStore{FirstSeen: start + 1000, OkValue: true},
EventBus: eventBus,
}
blamer, err := New(storages)
assert.Nil(t, err)
// Check when completely untriaged
blameLists, _ := blamer.GetAllBlameLists()
assert.NotNil(t, blameLists)
assert.Equal(t, 3, len(blameLists))
assert.Equal(t, 3, len(blameLists["foo"]))
assert.Equal(t, []int{1, 0, 0, 0}, blameLists["foo"][DI_1].Freq)
assert.Equal(t, []int{1, 0}, blameLists["foo"][DI_2].Freq)
assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["foo"][DI_3].Freq)
assert.Equal(t, 3, len(blameLists["bar"]))
assert.Equal(t, []int{2, 0, 0, 0}, blameLists["bar"][DI_4].Freq)
assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["bar"][DI_5].Freq)
assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["bar"][DI_6].Freq)
assert.Equal(t, &BlameDistribution{Freq: []int{1}}, blamer.GetBlame("foo", DI_1, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{3}}, blamer.GetBlame("foo", DI_2, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("foo", DI_3, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{1}}, blamer.GetBlame("bar", DI_4, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("bar", DI_5, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("bar", DI_6, commits))
// Classify some digests and re-calculate.
changes := map[string]types.TestClassification{
"foo": map[string]types.Label{DI_1: types.POSITIVE, DI_2: types.NEGATIVE},
"bar": map[string]types.Label{DI_4: types.POSITIVE, DI_6: types.NEGATIVE},
}
assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
// Wait for the change to propagate.
waitForChange(blamer, blameLists)
blameLists, _ = blamer.GetAllBlameLists()
assert.Equal(t, 3, len(blameLists))
assert.Equal(t, 1, len(blameLists["foo"]))
assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["foo"][DI_3].Freq)
assert.Equal(t, 1, len(blameLists["bar"]))
assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["bar"][DI_5].Freq)
assert.Equal(t, []int{1, 2, 0}, blameLists["baz"][DI_8].Freq)
assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("foo", DI_3, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("bar", DI_5, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{3}}, blamer.GetBlame("baz", DI_8, commits))
// Change the underlying tile and trigger with another change.
tile := storages.MasterTileBuilder.GetTile()
// Get the trace for the last parameters and set a value.
gTrace := tile.Traces[mocks.TraceKey(params[5])].(*types.GoldenTrace)
gTrace.Values[2] = DI_9
assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
// Wait for the change to propagate.
waitForChange(blamer, blameLists)
blameLists, _ = blamer.GetAllBlameLists()
assert.Equal(t, 3, len(blameLists))
assert.Equal(t, 1, len(blameLists["foo"]))
assert.Equal(t, 2, len(blameLists["bar"]))
assert.Equal(t, []int{1, 0, 0}, blameLists["bar"][DI_9].Freq)
assert.Equal(t, &BlameDistribution{Freq: []int{2}}, blamer.GetBlame("bar", DI_9, commits))
// Simulate the case where the digest is not found in digest store.
storages.DigestStore.(*mocks.MockDigestStore).OkValue = false
assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
time.Sleep(10 * time.Millisecond)
blameLists, _ = blamer.GetAllBlameLists()
assert.Equal(t, 3, len(blameLists))
assert.Equal(t, 1, len(blameLists["foo"]))
assert.Equal(t, 2, len(blameLists["bar"]))
assert.Equal(t, []int{1, 0, 0}, blameLists["bar"][DI_9].Freq)
assert.Equal(t, &BlameDistribution{Freq: []int{2}}, blamer.GetBlame("bar", DI_9, commits))
assert.Equal(t, &BlameDistribution{Freq: []int{1}}, blamer.GetBlame("bar", DI_9, commits[1:4]))
assert.Equal(t, &BlameDistribution{Freq: []int{}}, blamer.GetBlame("bar", DI_9, commits[0:2]))
}
func testStatusWatcher(t assert.TestingT, tileBuilder tracedb.MasterTileBuilder) {
eventBus := eventbus.New(nil)
storages := &storage.Storage{
ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
MasterTileBuilder: tileBuilder,
DigestStore: &MockDigestStore{},
EventBus: eventBus,
}
watcher, err := New(storages)
assert.Nil(t, err)
// Go through all corpora and change all the Items to positive.
status := watcher.GetStatus()
assert.NotNil(t, status)
for idx, corpStatus := range status.CorpStatus {
// Make sure no digests has any issues attached.
storages.DigestStore.(*MockDigestStore).issueIDs = nil
assert.False(t, corpStatus.OK)
tile, err := storages.GetLastTileTrimmed(true)
assert.Nil(t, err)
changes := map[string]types.TestClassification{}
posOrNeg := []types.Label{types.POSITIVE, types.NEGATIVE}
for _, trace := range tile.Traces {
if trace.Params()[types.CORPUS_FIELD] == corpStatus.Name {
gTrace := trace.(*types.GoldenTrace)
testName := gTrace.Params()[types.PRIMARY_KEY_FIELD]
for _, digest := range gTrace.Values {
if _, ok := changes[testName]; !ok {
changes[testName] = map[string]types.Label{}
}
changes[testName][digest] = posOrNeg[rand.Int()%2]
}
}
}
// Update the expecations and wait for the status to change.
assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
time.Sleep(1 * time.Second)
newStatus := watcher.GetStatus()
assert.False(t, newStatus.CorpStatus[idx].OK)
assert.False(t, newStatus.OK)
// Make sure all tests have an issue attached to each DigestInfo and
// trigger another expectations update.
storages.DigestStore.(*MockDigestStore).issueIDs = []int{1}
assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
time.Sleep(1 * time.Second)
// Make sure the current corpus is now ok.
newStatus = watcher.GetStatus()
assert.True(t, newStatus.CorpStatus[idx].OK)
}
// All corpora are ok therefore the overall status should be ok.
newStatus := watcher.GetStatus()
assert.True(t, newStatus.OK)
}
func testBlamerWithLiveData(t assert.TestingT, tileBuilder tracedb.MasterTileBuilder) {
eventBus := eventbus.New(nil)
storage := &storage.Storage{
ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
MasterTileBuilder: tileBuilder,
DigestStore: &mocks.MockDigestStore{
FirstSeen: time.Now().Unix(),
OkValue: true,
},
EventBus: eventBus,
}
blamer, err := New(storage)
assert.Nil(t, err)
// Wait until we have a blamelist.
var blameLists map[string]map[string]*BlameDistribution
for {
blameLists, _ = blamer.GetAllBlameLists()
if blameLists != nil {
break
}
}
tile := storage.MasterTileBuilder.GetTile()
// Since we set the 'First' timestamp of all digest info entries
// to Now. We should get a non-empty blamelist of all digests.
oneTestName := ""
oneDigest := ""
forEachTestDigestDo(tile, func(testName, digest string) {
assert.NotNil(t, blameLists[testName][digest])
assert.True(t, len(blameLists[testName][digest].Freq) > 0)
// Remember the last one for later.
oneTestName, oneDigest = testName, digest
})
// Change the classification of one test and trigger the recalculation.
changes := map[string]types.TestClassification{
oneTestName: map[string]types.Label{oneDigest: types.POSITIVE},
}
assert.Nil(t, storage.ExpectationsStore.AddChange(changes, ""))
// Wait for change to propagate.
waitForChange(blamer, blameLists)
blameLists, _ = blamer.GetAllBlameLists()
// Assert the correctness of the blamelists.
forEachTestDigestDo(tile, func(testName, digest string) {
if (testName == oneTestName) && (digest == oneDigest) {
assert.Nil(t, blameLists[testName][digest])
} else {
assert.NotNil(t, blameLists[testName][digest])
assert.True(t, len(blameLists[testName][digest].Freq) > 0)
}
})
// Set 'First' for all digests in the past and trigger another
// calculation.
storage.DigestStore.(*mocks.MockDigestStore).FirstSeen = 0
assert.Nil(t, storage.ExpectationsStore.AddChange(changes, ""))
waitForChange(blamer, blameLists)
blameLists, _ = blamer.GetAllBlameLists()
// Randomly assign labels to the different digests and make sure
// that the blamelists are correct.
storage.DigestStore.(*mocks.MockDigestStore).FirstSeen = time.Now().Unix()
changes = map[string]types.TestClassification{}
choices := []types.Label{types.POSITIVE, types.NEGATIVE, types.UNTRIAGED}
forEachTestDigestDo(tile, func(testName, digest string) {
targetTest := changes[testName]
if targetTest == nil {
targetTest = map[string]types.Label{}
}
// Randomly skip some digests.
label := choices[rand.Int()%len(choices)]
if label != types.UNTRIAGED {
targetTest[digest] = label
}
})
// Add the labels and wait for the recalculation.
assert.Nil(t, storage.ExpectationsStore.AddChange(changes, ""))
waitForChange(blamer, blameLists)
blameLists, commits := blamer.GetAllBlameLists()
expecations, err := storage.ExpectationsStore.Get()
assert.Nil(t, err)
// Verify that the results are plausible.
forEachTestTraceDo(tile, func(testName string, values []string) {
for idx, digest := range values {
if digest != types.MISSING_DIGEST {
label := expecations.Classification(testName, digest)
if label == types.UNTRIAGED {
bl := blameLists[testName][digest]
assert.NotNil(t, bl)
freq := bl.Freq
assert.True(t, len(freq) > 0)
startIdx := len(commits) - len(freq)
assert.True(t, (startIdx >= 0) && (startIdx <= idx), fmt.Sprintf("Expected (%s): Smaller than %d but got %d.", digest, startIdx, idx))
}
}
}
})
}