Пример #1
0
/**
  Conditions to test.

  Traces
  ------
  id   | config  | test name | corpus(source_type) |  digests
  a      8888      foo         gm                      aaa+, bbb-
  b      565       foo         gm                      ccc?, ddd?
  c      gpu       foo         gm                      eee+
  d      8888      bar         gm                      fff-, ggg?
  e      8888      quux        image                   jjj?

  Expectations
  ------------
  foo  aaa  pos
  foo  bbb  neg
  foo  ccc  unt
  foo  ddd  unt
  foo  eee  pos
  bar  fff  neg

  Ignores
  -------
  config=565

  Note no entry for quux or ggg, meaning untriaged.

  Test the following conditions and make sure you get
  the expected test summaries.

  source_type=gm
    foo - pos(aaa, eee):2  neg(bbb):1
    bar -                  neg(fff):1   unt(ggg):1

  source_type=gm includeIgnores=true
    foo - pos(aaa, eee):2  neg(bbb):1   unt(ccc, ddd):2
    bar -                  neg(fff):1   unt(ggg):1

  source_type=gm includeIgnores=true testName=foo
    foo - pos(aaa, eee):2  neg(bbb):1   unt(ccc, ddd):2

  testname = foo
    foo - pos(aaa, eee):2  neg(bbb):1

  testname = quux
    quux -                              unt(jjj):1

  config=565&config=8888
    foo - pos(aaa):1       neg(bbb):1
    bar -                  neg(fff):1   unt(ggg):1
    quux -                              unt(jjj):1

  config=565&config=8888 head=true
    foo -                  neg(bbb):1
    bar -                               unt(ggg):1
    quux -                              unt(jjj):1

  config=gpu
    foo - pos(eee):1

  config=unknown
    <empty>

*/
func TestCalcSummaries(t *testing.T) {
	tile := &tiling.Tile{
		Traces: map[string]tiling.Trace{
			"a": &types.GoldenTrace{
				Values: []string{"aaa", "bbb"},
				Params_: map[string]string{
					"name":        "foo",
					"config":      "8888",
					"source_type": "gm"},
			},
			"b": &types.GoldenTrace{
				Values: []string{"ccc", "ddd"},
				Params_: map[string]string{
					"name":        "foo",
					"config":      "565",
					"source_type": "gm"},
			},
			"c": &types.GoldenTrace{
				Values: []string{"eee", types.MISSING_DIGEST},
				Params_: map[string]string{
					"name":        "foo",
					"config":      "gpu",
					"source_type": "gm"},
			},
			"d": &types.GoldenTrace{
				Values: []string{"fff", "ggg"},
				Params_: map[string]string{
					"name":        "bar",
					"config":      "8888",
					"source_type": "gm"},
			},
			"e": &types.GoldenTrace{
				Values: []string{"jjj", types.MISSING_DIGEST},
				Params_: map[string]string{
					"name":        "quux",
					"config":      "8888",
					"source_type": "image"},
			},
		},
		Commits: []*tiling.Commit{
			&tiling.Commit{
				CommitTime: 42,
				Hash:       "ffffffffffffffffffffffffffffffffffffffff",
				Author:     "*****@*****.**",
			},
			&tiling.Commit{
				CommitTime: 45,
				Hash:       "gggggggggggggggggggggggggggggggggggggggg",
				Author:     "*****@*****.**",
			},
		},
		Scale:     0,
		TileIndex: 0,
	}

	eventBus := eventbus.New(nil)
	storages := &storage.Storage{
		DiffStore:         mocks.MockDiffStore{},
		ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
		IgnoreStore:       ignore.NewMemIgnoreStore(),
		MasterTileBuilder: mocks.NewMockTileBuilderFromTile(t, tile),
		NCommits:          50,
		EventBus:          eventBus,
		DigestStore:       &mocks.MockDigestStore{FirstSeen: time.Now().Unix() + 1000, OkValue: true},
	}

	assert.Nil(t, storages.ExpectationsStore.AddChange(map[string]types.TestClassification{
		"foo": map[string]types.Label{
			"aaa": types.POSITIVE,
			"bbb": types.NEGATIVE,
			"ccc": types.UNTRIAGED,
			"ddd": types.UNTRIAGED,
			"eee": types.POSITIVE,
		},
		"bar": map[string]types.Label{
			"fff": types.NEGATIVE,
		},
	}, "*****@*****.**"))

	ta, _ := tally.New(storages)
	assert.Nil(t, storages.IgnoreStore.Create(&ignore.IgnoreRule{
		ID:      1,
		Name:    "foo",
		Expires: time.Now().Add(time.Hour),
		Query:   "config=565",
	}))

	blamer, err := blame.New(storages)
	assert.Nil(t, err)

	summaries, err := New(storages, ta, blamer)
	assert.Nil(t, err)

	sum, err := summaries.CalcSummaries(nil, "source_type=gm", false, false)
	if err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 2, len(sum))
	triageCountsCorrect(t, sum, "foo", 2, 1, 0)
	triageCountsCorrect(t, sum, "bar", 0, 1, 1)
	assert.Equal(t, []string{}, sum["foo"].UntHashes)
	assert.Equal(t, []string{"ggg"}, sum["bar"].UntHashes)

	if sum, err = summaries.CalcSummaries(nil, "source_type=gm", true, false); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 2, len(sum))
	triageCountsCorrect(t, sum, "foo", 2, 1, 2)
	triageCountsCorrect(t, sum, "bar", 0, 1, 1)
	assert.Equal(t, sum["foo"].UntHashes, []string{"ccc", "ddd"})
	assert.Equal(t, sum["bar"].UntHashes, []string{"ggg"})

	if sum, err = summaries.CalcSummaries([]string{"foo"}, "source_type=gm", true, false); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 1, len(sum))
	triageCountsCorrect(t, sum, "foo", 2, 1, 2)
	assert.Equal(t, sum["foo"].UntHashes, []string{"ccc", "ddd"})

	if sum, err = summaries.CalcSummaries([]string{"foo"}, "", false, false); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 1, len(sum))
	triageCountsCorrect(t, sum, "foo", 2, 1, 0)
	assert.Equal(t, sum["foo"].UntHashes, []string{})

	if sum, err = summaries.CalcSummaries(nil, "config=8888&config=565", false, false); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 3, len(sum))
	triageCountsCorrect(t, sum, "foo", 1, 1, 0)
	triageCountsCorrect(t, sum, "bar", 0, 1, 1)
	triageCountsCorrect(t, sum, "quux", 0, 0, 1)
	assert.Equal(t, sum["foo"].UntHashes, []string{})
	assert.Equal(t, sum["bar"].UntHashes, []string{"ggg"})
	assert.Equal(t, sum["quux"].UntHashes, []string{"jjj"})

	if sum, err = summaries.CalcSummaries(nil, "config=8888&config=565", false, true); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 3, len(sum))
	triageCountsCorrect(t, sum, "foo", 0, 1, 0)
	triageCountsCorrect(t, sum, "bar", 0, 0, 1)
	triageCountsCorrect(t, sum, "quux", 0, 0, 1)
	assert.Equal(t, sum["foo"].UntHashes, []string{})
	assert.Equal(t, sum["bar"].UntHashes, []string{"ggg"})
	assert.Equal(t, sum["quux"].UntHashes, []string{"jjj"})

	if sum, err = summaries.CalcSummaries(nil, "config=gpu", false, false); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 1, len(sum))
	triageCountsCorrect(t, sum, "foo", 1, 0, 0)
	assert.Equal(t, sum["foo"].UntHashes, []string{})

	if sum, err = summaries.CalcSummaries(nil, "config=unknown", false, false); err != nil {
		t.Fatalf("Failed to calc: %s", err)
	}
	assert.Equal(t, 0, len(sum))
}
Пример #2
0
func TestBlamerWithSyntheticData(t *testing.T) {
	start := time.Now().Unix()
	commits := []*tiling.Commit{
		&tiling.Commit{CommitTime: start + 10, Hash: "h1", Author: "John Doe 1"},
		&tiling.Commit{CommitTime: start + 20, Hash: "h2", Author: "John Doe 2"},
		&tiling.Commit{CommitTime: start + 30, Hash: "h3", Author: "John Doe 3"},
		&tiling.Commit{CommitTime: start + 40, Hash: "h4", Author: "John Doe 4"},
		&tiling.Commit{CommitTime: start + 50, Hash: "h5", Author: "John Doe 5"},
	}

	params := []map[string]string{
		map[string]string{"name": "foo", "config": "8888", "source_type": "gm"},
		map[string]string{"name": "foo", "config": "565", "source_type": "gm"},
		map[string]string{"name": "foo", "config": "gpu", "source_type": "gm"},
		map[string]string{"name": "bar", "config": "8888", "source_type": "gm"},
		map[string]string{"name": "bar", "config": "565", "source_type": "gm"},
		map[string]string{"name": "bar", "config": "gpu", "source_type": "gm"},
		map[string]string{"name": "baz", "config": "565", "source_type": "gm"},
		map[string]string{"name": "baz", "config": "gpu", "source_type": "gm"},
	}

	DI_1, DI_2, DI_3 := "digest1", "digest2", "digest3"
	DI_4, DI_5, DI_6 := "digest4", "digest5", "digest6"
	DI_7, DI_8, DI_9 := "digest7", "digest8", "digest9"
	MISS := types.MISSING_DIGEST

	digests := [][]string{
		[]string{MISS, MISS, DI_1, MISS, MISS},
		[]string{MISS, DI_1, DI_1, DI_2, MISS},
		[]string{DI_3, MISS, MISS, MISS, MISS},
		[]string{DI_5, DI_4, DI_5, DI_5, DI_5},
		[]string{DI_6, MISS, DI_4, MISS, MISS},
		[]string{MISS, MISS, MISS, MISS, MISS},
		[]string{DI_7, DI_7, MISS, DI_8, MISS},
		[]string{DI_7, MISS, DI_7, DI_8, MISS},
	}

	// Make sure the data are consistent and create a mock TileStore.
	assert.Equal(t, len(commits), len(digests[0]))
	assert.Equal(t, len(digests), len(params))

	eventBus := eventbus.New(nil)
	storages := &storage.Storage{
		ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
		MasterTileBuilder: mocks.NewMockTileBuilder(t, digests, params, commits),
		DigestStore:       &mocks.MockDigestStore{FirstSeen: start + 1000, OkValue: true},
		EventBus:          eventBus,
	}
	blamer, err := New(storages)
	assert.Nil(t, err)

	// Check when completely untriaged
	blameLists, _ := blamer.GetAllBlameLists()
	assert.NotNil(t, blameLists)

	assert.Equal(t, 3, len(blameLists))
	assert.Equal(t, 3, len(blameLists["foo"]))
	assert.Equal(t, []int{1, 0, 0, 0}, blameLists["foo"][DI_1].Freq)
	assert.Equal(t, []int{1, 0}, blameLists["foo"][DI_2].Freq)
	assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["foo"][DI_3].Freq)

	assert.Equal(t, 3, len(blameLists["bar"]))
	assert.Equal(t, []int{2, 0, 0, 0}, blameLists["bar"][DI_4].Freq)
	assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["bar"][DI_5].Freq)
	assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["bar"][DI_6].Freq)

	assert.Equal(t, &BlameDistribution{Freq: []int{1}}, blamer.GetBlame("foo", DI_1, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{3}}, blamer.GetBlame("foo", DI_2, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("foo", DI_3, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{1}}, blamer.GetBlame("bar", DI_4, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("bar", DI_5, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("bar", DI_6, commits))

	// Classify some digests and re-calculate.
	changes := map[string]types.TestClassification{
		"foo": map[string]types.Label{DI_1: types.POSITIVE, DI_2: types.NEGATIVE},
		"bar": map[string]types.Label{DI_4: types.POSITIVE, DI_6: types.NEGATIVE},
	}
	assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))

	// Wait for the change to propagate.
	waitForChange(blamer, blameLists)
	blameLists, _ = blamer.GetAllBlameLists()

	assert.Equal(t, 3, len(blameLists))
	assert.Equal(t, 1, len(blameLists["foo"]))
	assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["foo"][DI_3].Freq)

	assert.Equal(t, 1, len(blameLists["bar"]))
	assert.Equal(t, []int{1, 0, 0, 0, 0}, blameLists["bar"][DI_5].Freq)
	assert.Equal(t, []int{1, 2, 0}, blameLists["baz"][DI_8].Freq)

	assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("foo", DI_3, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{0}}, blamer.GetBlame("bar", DI_5, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{3}}, blamer.GetBlame("baz", DI_8, commits))

	// Change the underlying tile and trigger with another change.
	tile := storages.MasterTileBuilder.GetTile()

	// Get the trace for the last parameters and set a value.
	gTrace := tile.Traces[mocks.TraceKey(params[5])].(*types.GoldenTrace)
	gTrace.Values[2] = DI_9

	assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))

	// Wait for the change to propagate.
	waitForChange(blamer, blameLists)
	blameLists, _ = blamer.GetAllBlameLists()

	assert.Equal(t, 3, len(blameLists))
	assert.Equal(t, 1, len(blameLists["foo"]))
	assert.Equal(t, 2, len(blameLists["bar"]))
	assert.Equal(t, []int{1, 0, 0}, blameLists["bar"][DI_9].Freq)

	assert.Equal(t, &BlameDistribution{Freq: []int{2}}, blamer.GetBlame("bar", DI_9, commits))

	// Simulate the case where the digest is not found in digest store.
	storages.DigestStore.(*mocks.MockDigestStore).OkValue = false
	assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
	time.Sleep(10 * time.Millisecond)
	blameLists, _ = blamer.GetAllBlameLists()
	assert.Equal(t, 3, len(blameLists))
	assert.Equal(t, 1, len(blameLists["foo"]))
	assert.Equal(t, 2, len(blameLists["bar"]))
	assert.Equal(t, []int{1, 0, 0}, blameLists["bar"][DI_9].Freq)

	assert.Equal(t, &BlameDistribution{Freq: []int{2}}, blamer.GetBlame("bar", DI_9, commits))
	assert.Equal(t, &BlameDistribution{Freq: []int{1}}, blamer.GetBlame("bar", DI_9, commits[1:4]))
	assert.Equal(t, &BlameDistribution{Freq: []int{}}, blamer.GetBlame("bar", DI_9, commits[0:2]))
}
Пример #3
0
func testStatusWatcher(t assert.TestingT, tileBuilder tracedb.MasterTileBuilder) {
	eventBus := eventbus.New(nil)
	storages := &storage.Storage{
		ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
		MasterTileBuilder: tileBuilder,
		DigestStore:       &MockDigestStore{},
		EventBus:          eventBus,
	}

	watcher, err := New(storages)
	assert.Nil(t, err)

	// Go through all corpora and change all the Items to positive.
	status := watcher.GetStatus()
	assert.NotNil(t, status)

	for idx, corpStatus := range status.CorpStatus {
		// Make sure no digests has any issues attached.
		storages.DigestStore.(*MockDigestStore).issueIDs = nil

		assert.False(t, corpStatus.OK)
		tile, err := storages.GetLastTileTrimmed(true)
		assert.Nil(t, err)

		changes := map[string]types.TestClassification{}
		posOrNeg := []types.Label{types.POSITIVE, types.NEGATIVE}
		for _, trace := range tile.Traces {
			if trace.Params()[types.CORPUS_FIELD] == corpStatus.Name {
				gTrace := trace.(*types.GoldenTrace)
				testName := gTrace.Params()[types.PRIMARY_KEY_FIELD]
				for _, digest := range gTrace.Values {
					if _, ok := changes[testName]; !ok {
						changes[testName] = map[string]types.Label{}
					}
					changes[testName][digest] = posOrNeg[rand.Int()%2]
				}
			}
		}

		// Update the expecations and wait for the status to change.
		assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
		time.Sleep(1 * time.Second)
		newStatus := watcher.GetStatus()
		assert.False(t, newStatus.CorpStatus[idx].OK)
		assert.False(t, newStatus.OK)

		// Make sure all tests have an issue attached to each DigestInfo and
		// trigger another expectations update.
		storages.DigestStore.(*MockDigestStore).issueIDs = []int{1}
		assert.Nil(t, storages.ExpectationsStore.AddChange(changes, ""))
		time.Sleep(1 * time.Second)

		// Make sure the current corpus is now ok.
		newStatus = watcher.GetStatus()
		assert.True(t, newStatus.CorpStatus[idx].OK)
	}

	// All corpora are ok therefore the overall status should be ok.
	newStatus := watcher.GetStatus()
	assert.True(t, newStatus.OK)
}
Пример #4
0
func testBlamerWithLiveData(t assert.TestingT, tileBuilder tracedb.MasterTileBuilder) {
	eventBus := eventbus.New(nil)
	storage := &storage.Storage{
		ExpectationsStore: expstorage.NewMemExpectationsStore(eventBus),
		MasterTileBuilder: tileBuilder,
		DigestStore: &mocks.MockDigestStore{
			FirstSeen: time.Now().Unix(),
			OkValue:   true,
		},
		EventBus: eventBus,
	}

	blamer, err := New(storage)
	assert.Nil(t, err)

	// Wait until we have a blamelist.
	var blameLists map[string]map[string]*BlameDistribution
	for {
		blameLists, _ = blamer.GetAllBlameLists()
		if blameLists != nil {
			break
		}
	}

	tile := storage.MasterTileBuilder.GetTile()

	// Since we set the 'First' timestamp of all digest info entries
	// to Now. We should get a non-empty blamelist of all digests.
	oneTestName := ""
	oneDigest := ""
	forEachTestDigestDo(tile, func(testName, digest string) {
		assert.NotNil(t, blameLists[testName][digest])
		assert.True(t, len(blameLists[testName][digest].Freq) > 0)

		// Remember the last one for later.
		oneTestName, oneDigest = testName, digest
	})

	// Change the classification of one test and trigger the recalculation.
	changes := map[string]types.TestClassification{
		oneTestName: map[string]types.Label{oneDigest: types.POSITIVE},
	}
	assert.Nil(t, storage.ExpectationsStore.AddChange(changes, ""))

	// Wait for change to propagate.
	waitForChange(blamer, blameLists)
	blameLists, _ = blamer.GetAllBlameLists()

	// Assert the correctness of the blamelists.
	forEachTestDigestDo(tile, func(testName, digest string) {
		if (testName == oneTestName) && (digest == oneDigest) {
			assert.Nil(t, blameLists[testName][digest])
		} else {
			assert.NotNil(t, blameLists[testName][digest])
			assert.True(t, len(blameLists[testName][digest].Freq) > 0)
		}
	})

	// Set 'First' for all digests in the past and trigger another
	// calculation.
	storage.DigestStore.(*mocks.MockDigestStore).FirstSeen = 0
	assert.Nil(t, storage.ExpectationsStore.AddChange(changes, ""))
	waitForChange(blamer, blameLists)
	blameLists, _ = blamer.GetAllBlameLists()

	// Randomly assign labels to the different digests and make sure
	// that the blamelists are correct.
	storage.DigestStore.(*mocks.MockDigestStore).FirstSeen = time.Now().Unix()

	changes = map[string]types.TestClassification{}
	choices := []types.Label{types.POSITIVE, types.NEGATIVE, types.UNTRIAGED}
	forEachTestDigestDo(tile, func(testName, digest string) {
		targetTest := changes[testName]
		if targetTest == nil {
			targetTest = map[string]types.Label{}
		}
		// Randomly skip some digests.
		label := choices[rand.Int()%len(choices)]
		if label != types.UNTRIAGED {
			targetTest[digest] = label
		}
	})

	// Add the labels and wait for the recalculation.
	assert.Nil(t, storage.ExpectationsStore.AddChange(changes, ""))
	waitForChange(blamer, blameLists)
	blameLists, commits := blamer.GetAllBlameLists()

	expecations, err := storage.ExpectationsStore.Get()
	assert.Nil(t, err)

	// Verify that the results are plausible.
	forEachTestTraceDo(tile, func(testName string, values []string) {
		for idx, digest := range values {
			if digest != types.MISSING_DIGEST {
				label := expecations.Classification(testName, digest)
				if label == types.UNTRIAGED {
					bl := blameLists[testName][digest]
					assert.NotNil(t, bl)
					freq := bl.Freq
					assert.True(t, len(freq) > 0)
					startIdx := len(commits) - len(freq)
					assert.True(t, (startIdx >= 0) && (startIdx <= idx), fmt.Sprintf("Expected (%s): Smaller than %d but got %d.", digest, startIdx, idx))
				}
			}
		}
	})
}