// Tests the processor in conjunction with the vcs.
func TestGoldProcessor(t *testing.T) {
// Set up mock VCS and run a servcer with the given data directory.
vcs := ingestion.MockVCS(TEST_COMMITS)
server, serverAddr := ingestion.StartTraceDBTestServer(t, TRACE_DB_FILENAME)
defer server.Stop()
defer testutils.Remove(t, TRACE_DB_FILENAME)
ingesterConf := &sharedconfig.IngesterConfig{
ExtraParams: map[string]string{
CONFIG_TRACESERVICE: serverAddr,
},
}
// Set up the processor.
processor, err := newGoldProcessor(vcs, ingesterConf, nil)
assert.Nil(t, err)
// Load the example file and process it.
fsResult, err := ingestion.FileSystemResult(TEST_INGESTION_FILE, "./")
assert.Nil(t, err)
err = processor.Process(fsResult)
assert.Nil(t, err)
// Steal the traceDB used by the processor to verify the results.
traceDB := processor.(*goldProcessor).traceDB
startTime := time.Now().Add(-time.Hour * 24 * 10)
commitIDs, err := traceDB.List(startTime, time.Now())
assert.Nil(t, err)
assert.Equal(t, 1, len(FilterCommitIDs(commitIDs, "master")))
assert.Equal(t, 0, len(FilterCommitIDs(commitIDs, TEST_CODE_REVIEW_URL)))
assert.Equal(t, 1, len(commitIDs))
assert.Equal(t, &tracedb.CommitID{
Timestamp: TEST_COMMITS[0].Timestamp.Unix(),
ID: TEST_COMMITS[0].Hash,
Source: "master",
}, commitIDs[0])
// Get a tile and make sure we have the right number of traces.
tile, _, err := traceDB.TileFromCommits(commitIDs)
assert.Nil(t, err)
traces := tile.Traces
assert.Equal(t, len(TEST_ENTRIES), len(traces))
for _, testEntry := range TEST_ENTRIES {
found, ok := traces[testEntry.key]
assert.True(t, ok)
goldTrace, ok := found.(*types.GoldenTrace)
assert.True(t, ok)
assert.Equal(t, 1, len(goldTrace.Values))
assert.Equal(t, testEntry.value, goldTrace.Values[0])
}
assert.Equal(t, "master", commitIDs[0].Source)
assert.Nil(t, traceDB.Close())
}
func TestTrybotPerfIngestion(t *testing.T) {
b, err := ioutil.ReadFile(filepath.Join("testdata", "rietveld_response.txt"))
assert.NoError(t, err)
m := mockhttpclient.NewURLMock()
m.Mock("https://codereview.chromium.org/api/1467533002/1", b)
server, serverAddr := ingestion.StartTraceDBTestServer(t, "./trybot_test_trace.db")
defer server.Stop()
defer testutils.Remove(t, "./trybot_test_trace.db")
ingesterConf := &sharedconfig.IngesterConfig{
ExtraParams: map[string]string{
CONFIG_TRACESERVICE: serverAddr,
},
}
processor, err := newPerfTrybotProcessor(nil, ingesterConf, nil)
assert.Nil(t, err)
processor.(*perfTrybotProcessor).review = rietveld.New("https://codereview.chromium.org", m.Client())
fsResult, err := ingestion.FileSystemResult(filepath.Join(TEST_DATA_DIR, "trybot.json"), TEST_DATA_DIR)
assert.Nil(t, err)
err = processor.Process(fsResult)
assert.Nil(t, err)
assert.Equal(t, 1, len(processor.(*perfTrybotProcessor).cache))
// Steal the traceDB used by the processor to verify the results.
traceDB := processor.(*perfTrybotProcessor).traceDB
startTime := time.Time{}
commitIDs, err := traceDB.List(startTime, time.Now())
assert.Nil(t, err)
assert.Equal(t, 1, len(commitIDs))
assert.Equal(t, &tracedb.CommitID{
Timestamp: 1448036640,
ID: "1",
Source: "https://codereview.chromium.org/1467533002",
}, commitIDs[0])
// Get a tile and make sure we have the right number of traces.
tile, _, err := traceDB.TileFromCommits(commitIDs)
assert.Nil(t, err)
traces := tile.Traces
assert.Equal(t, 2, len(traces))
assert.Nil(t, traceDB.Close())
}
func TestPDFProcessor(t *testing.T) {
testutils.SkipIfShort(t)
// Get the service account client from meta data or a local config file.
client, err := auth.NewJWTServiceAccountClient("", auth.DEFAULT_JWT_FILENAME, nil, storage.ScopeFullControl)
assert.Nil(t, err)
cacheDir, err := fileutil.EnsureDirExists(CACHE_DIR)
assert.Nil(t, err)
// Clean up after the test.
defer func() {
defer util.RemoveAll(cacheDir)
deleteFolderContent(t, TEST_BUCKET, IMAGES_OUT_DIR, client)
deleteFolderContent(t, TEST_BUCKET, JSON_OUT_DIR, client)
}()
// Configure the processor.
ingesterConf := &sharedconfig.IngesterConfig{
ExtraParams: map[string]string{
CONFIG_INPUT_IMAGES_BUCKET: TEST_BUCKET,
CONFIG_INPUT_IMAGES_DIR: IMAGES_IN_DIR,
CONFIG_OUTPUT_JSON_BUCKET: TEST_BUCKET,
CONFIG_OUTPUT_JSON_DIR: JSON_OUT_DIR,
CONFIG_OUTPUT_IMAGES_BUCKET: TEST_BUCKET,
CONFIG_OUTPUT_IMAGES_DIR: IMAGES_OUT_DIR,
CONFIG_PDF_CACHEDIR: cacheDir,
},
}
processor, err := newPDFProcessor(nil, ingesterConf, client)
assert.Nil(t, err)
// Load the example file and process it.
fsResult, err := ingestion.FileSystemResult(TEST_INGESTION_FILE, "./")
assert.Nil(t, err)
err = processor.Process(fsResult)
assert.Nil(t, err)
// Fetch the json output and parse it.
pProcessor := processor.(*pdfProcessor)
// download the result.
resultFileName := filepath.Join(CACHE_DIR, "result-file.json")
assert.Nil(t, pProcessor.download(TEST_BUCKET, JSON_OUT_DIR, fsResult.Name(), resultFileName))
// Make sure we get the expected result.
fsResult, err = ingestion.FileSystemResult(TEST_INGESTION_FILE, "./")
assert.Nil(t, err)
r, err := fsResult.Open()
assert.Nil(t, err)
fsDMResults, err := goldingestion.ParseDMResultsFromReader(r)
assert.Nil(t, err)
foundResult, err := ingestion.FileSystemResult(resultFileName, "./")
assert.Nil(t, err)
r, err = foundResult.Open()
assert.Nil(t, err)
foundDMResults, err := goldingestion.ParseDMResultsFromReader(r)
assert.Nil(t, err)
dmResult1 := *fsDMResults
dmResult2 := *foundDMResults
dmResult1.Results = nil
dmResult2.Results = nil
assert.Equal(t, dmResult1, dmResult2)
foundIdx := 0
srcResults := fsDMResults.Results
tgtResults := foundDMResults.Results
for _, result := range srcResults {
assert.True(t, foundIdx < len(tgtResults))
if result.Options["ext"] == "pdf" {
for ; (foundIdx < len(tgtResults)) && (result.Key["name"] == tgtResults[foundIdx].Key["name"]); foundIdx++ {
assert.True(t, tgtResults[foundIdx].Key["rasterizer"] != "")
delete(tgtResults[foundIdx].Key, "rasterizer")
assert.Equal(t, result.Key, tgtResults[foundIdx].Key)
assert.Equal(t, "png", tgtResults[foundIdx].Options["ext"])
}
}
}
assert.Equal(t, len(foundDMResults.Results), foundIdx)
}
// Tests the processor in conjunction with the vcs.
func TestTrybotGoldProcessor(t *testing.T) {
// Set up mock VCS and run a servcer with the given data directory.
vcs := ingestion.MockVCS(TEST_COMMITS)
server, serverAddr := ingestion.StartTraceDBTestServer(t, TRYBOT_TRACE_DB_FILE)
defer server.Stop()
defer testutils.Remove(t, TRYBOT_TRACE_DB_FILE)
ingesterConf := &sharedconfig.IngesterConfig{
ExtraParams: map[string]string{
CONFIG_TRACESERVICE: serverAddr,
CONFIG_CODE_REVIEW_URL: TEST_CODE_REVIEW_URL,
},
}
// Set up the processor.
processor, err := newGoldTrybotProcessor(vcs, ingesterConf)
assert.Nil(t, err)
// Load the example file and process it.
fsResult, err := ingestion.FileSystemResult(TRYBOT_INGESTION_FILE)
assert.Nil(t, err)
err = processor.Process(fsResult)
assert.Nil(t, err)
// Steal the traceDB used by the processor to verify the results.
traceDB := processor.(*goldTrybotProcessor).traceDB
// The timestamp for the issue/patchset in the testfile is 1443718869.
startTime := time.Unix(1443718868, 0)
commitIDs, err := traceDB.List(startTime, time.Now())
assert.Nil(t, err)
assert.Equal(t, 1, len(FilterCommitIDs(commitIDs, TRYBOT_SRC)))
assert.Equal(t, 0, len(FilterCommitIDs(commitIDs, VCS_SRC)))
assert.Equal(t, 1, len(commitIDs))
assert.Equal(t, &tracedb.CommitID{
Timestamp: time.Unix(1443718869, 0),
ID: "1",
Source: string(TRYBOT_SRC) + TEST_CODE_REVIEW_URL + "/1381483003",
}, commitIDs[0])
// Get a tile and make sure we have the right number of traces.
tile, err := traceDB.TileFromCommits(commitIDs)
assert.Nil(t, err)
traces := tile.Traces
assert.Equal(t, len(TEST_ENTRIES), len(traces))
for _, testEntry := range TEST_ENTRIES {
found, ok := traces[testEntry.key]
assert.True(t, ok)
goldTrace, ok := found.(*types.GoldenTrace)
assert.True(t, ok)
assert.Equal(t, 1, len(goldTrace.Values))
assert.Equal(t, testEntry.value, goldTrace.Values[0])
}
// Make sure the prefix is stripped correctly.
StripSourcePrefix(commitIDs)
assert.Equal(t, TEST_CODE_REVIEW_URL+"/1381483003", commitIDs[0].Source)
assert.Nil(t, traceDB.Close())
}