// retrieveFile retrieves a single file (or directory) from a Docker container.
func retrieveFile(target *core.BuildTarget, cid []byte, filename string, warn bool) {
log.Debug("Attempting to retrieve file %s for %s...", filename, target.Label)
timeout := core.State.Config.Docker.ResultsTimeout
cmd := []string{"docker", "cp", string(cid) + ":/tmp/test/" + filename, target.TestDir()}
if out, err := core.ExecWithTimeoutSimple(timeout, cmd...); err != nil {
if warn {
log.Warning("Failed to retrieve results for %s: %s [%s]", target.Label, err, out)
} else {
log.Debug("Failed to retrieve results for %s: %s [%s]", target.Label, err, out)
}
}
}
func runTest(state *core.BuildState, target *core.BuildTarget) ([]byte, error) {
replacedCmd := build.ReplaceTestSequences(target, target.GetTestCommand())
env := core.BuildEnvironment(state, target, true)
if len(state.TestArgs) > 0 {
args := strings.Join(state.TestArgs, " ")
replacedCmd += " " + args
env = append(env, "TESTS="+args)
}
log.Debug("Running test %s\nENVIRONMENT:\n%s\n%s", target.Label, strings.Join(env, "\n"), replacedCmd)
_, out, err := core.ExecWithTimeoutShell(target.TestDir(), env, target.TestTimeout, state.Config.Test.Timeout, state.ShowAllOutput, replacedCmd)
return out, err
}
func prepareTestDir(graph *core.BuildGraph, target *core.BuildTarget) error {
if err := os.RemoveAll(target.TestDir()); err != nil {
return err
}
if err := os.MkdirAll(target.TestDir(), core.DirPermissions); err != nil {
return err
}
for out := range core.IterRuntimeFiles(graph, target, true) {
if err := core.PrepareSourcePair(out); err != nil {
return err
}
}
return nil
}
func runContainerisedTest(state *core.BuildState, target *core.BuildTarget) ([]byte, error) {
testDir := path.Join(core.RepoRoot, target.TestDir())
replacedCmd := build.ReplaceTestSequences(target, target.GetTestCommand())
replacedCmd += " " + strings.Join(state.TestArgs, " ")
containerName := state.Config.Docker.DefaultImage
if target.ContainerSettings != nil && target.ContainerSettings.DockerImage != "" {
containerName = target.ContainerSettings.DockerImage
}
// Gentle hack: remove the absolute path from the command
replacedCmd = strings.Replace(replacedCmd, testDir, "/tmp/test", -1)
// Fiddly hack follows to handle docker run --rm failing saying "Cannot destroy container..."
// "Driver aufs failed to remove root filesystem... device or resource busy"
cidfile := path.Join(testDir, ".container_id")
// Using C.UTF-8 for LC_ALL because it works. Not sure it's strictly
// correct to mix that with LANG=en_GB.UTF-8
command := []string{"docker", "run", "--cidfile", cidfile, "-e", "LC_ALL=C.UTF-8"}
if target.ContainerSettings != nil {
if target.ContainerSettings.DockerRunArgs != "" {
command = append(command, strings.Split(target.ContainerSettings.DockerRunArgs, " ")...)
}
if target.ContainerSettings.DockerUser != "" {
command = append(command, "-u", target.ContainerSettings.DockerUser)
}
} else {
command = append(command, state.Config.Docker.RunArgs...)
}
for _, env := range core.BuildEnvironment(state, target, true) {
command = append(command, "-e", strings.Replace(env, testDir, "/tmp/test", -1))
}
replacedCmd = "mkdir -p /tmp/test && cp -r /tmp/test_in/* /tmp/test && cd /tmp/test && " + replacedCmd
command = append(command, "-v", testDir+":/tmp/test_in", "-w", "/tmp/test_in", containerName, "bash", "-o", "pipefail", "-c", replacedCmd)
log.Debug("Running containerised test %s: %s", target.Label, strings.Join(command, " "))
_, out, err := core.ExecWithTimeout(target.TestDir(), nil, target.TestTimeout, state.Config.Test.Timeout, state.ShowAllOutput, command)
retrieveResultsAndRemoveContainer(target, cidfile, err == context.DeadlineExceeded)
return out, err
}
func test(tid int, state *core.BuildState, label core.BuildLabel, target *core.BuildTarget) {
startTime := time.Now()
hash, err := build.RuntimeHash(state, target)
if err != nil {
state.LogBuildError(tid, label, core.TargetTestFailed, err, "Failed to calculate target hash")
return
}
// Check the cached output files if the target wasn't rebuilt.
hash = core.CollapseHash(hash)
hashStr := base64.RawURLEncoding.EncodeToString(hash)
resultsFileName := fmt.Sprintf(".test_results_%s_%s", label.Name, hashStr)
coverageFileName := fmt.Sprintf(".test_coverage_%s_%s", label.Name, hashStr)
outputFile := path.Join(target.TestDir(), "test.results")
coverageFile := path.Join(target.TestDir(), "test.coverage")
cachedOutputFile := path.Join(target.OutDir(), resultsFileName)
cachedCoverageFile := path.Join(target.OutDir(), coverageFileName)
needCoverage := state.NeedCoverage && !target.NoTestOutput
cachedTest := func() {
log.Debug("Not re-running test %s; got cached results.", label)
coverage := parseCoverageFile(target, cachedCoverageFile)
results, err := parseTestResults(target, cachedOutputFile, true)
target.Results.Duration = time.Since(startTime).Seconds()
target.Results.Cached = true
if err != nil {
state.LogBuildError(tid, label, core.TargetTestFailed, err, "Failed to parse cached test file %s", cachedOutputFile)
} else if results.Failed > 0 {
panic("Test results with failures shouldn't be cached.")
} else {
logTestSuccess(state, tid, label, results, coverage)
}
}
moveAndCacheOutputFiles := func(results core.TestResults, coverage core.TestCoverage) bool {
// Never cache test results when given arguments; the results may be incomplete.
if len(state.TestArgs) > 0 {
log.Debug("Not caching results for %s, we passed it arguments", label)
return true
}
if err := moveAndCacheOutputFile(state, target, hash, outputFile, cachedOutputFile, resultsFileName, dummyOutput); err != nil {
state.LogTestResult(tid, label, core.TargetTestFailed, results, coverage, err, "Failed to move test output file")
return false
}
if needCoverage || core.PathExists(coverageFile) {
if err := moveAndCacheOutputFile(state, target, hash, coverageFile, cachedCoverageFile, coverageFileName, dummyCoverage); err != nil {
state.LogTestResult(tid, label, core.TargetTestFailed, results, coverage, err, "Failed to move test coverage file")
return false
}
}
for _, output := range target.TestOutputs {
tmpFile := path.Join(target.TestDir(), output)
outFile := path.Join(target.OutDir(), output)
if err := moveAndCacheOutputFile(state, target, hash, tmpFile, outFile, output, ""); err != nil {
state.LogTestResult(tid, label, core.TargetTestFailed, results, coverage, err, "Failed to move test output file")
return false
}
}
return true
}
needToRun := func() bool {
if target.State() == core.Unchanged && core.PathExists(cachedOutputFile) {
// Output file exists already and appears to be valid. We might still need to rerun though
// if the coverage files aren't available.
if needCoverage && !core.PathExists(cachedCoverageFile) {
return true
}
return false
}
// Check the cache for these artifacts.
if state.Cache == nil {
return true
}
cache := *state.Cache
if !cache.RetrieveExtra(target, hash, resultsFileName) {
return true
}
if needCoverage && !cache.RetrieveExtra(target, hash, coverageFileName) {
return true
}
for _, output := range target.TestOutputs {
if !cache.RetrieveExtra(target, hash, output) {
return true
}
}
return false
}
// Don't cache when doing multiple runs, presumably the user explicitly wants to check it.
if state.NumTestRuns <= 1 && !needToRun() {
cachedTest()
return
}
// Remove any cached test result file.
if err := RemoveCachedTestFiles(target); err != nil {
state.LogBuildError(tid, label, core.TargetTestFailed, err, "Failed to remove cached test files")
return
}
numSucceeded := 0
numFlakes := 0
numRuns, successesRequired := calcNumRuns(state.NumTestRuns, target.Flakiness)
var resultErr error
resultMsg := ""
var coverage core.TestCoverage
for i := 0; i < numRuns && numSucceeded < successesRequired; i++ {
if numRuns > 1 {
state.LogBuildResult(tid, label, core.TargetTesting, fmt.Sprintf("Testing (%d of %d)...", i+1, numRuns))
}
out, err := prepareAndRunTest(tid, state, target)
duration := time.Since(startTime).Seconds()
startTime = time.Now() // reset this for next time
// This is all pretty involved; there are lots of different possibilities of what could happen.
// The contract is that the test must return zero on success or non-zero on failure (Unix FTW).
// If it's successful, it must produce a parseable file named "test.results" in its temp folder.
// (alternatively, this can be a directory containing parseable files).
// Tests can opt out of the file requirement individually, in which case they're judged only
// by their return value.
// But of course, we still have to consider all the alternatives here and handle them nicely.
target.Results.Output = string(out)
if err != nil && target.Results.Output == "" {
target.Results.Output = err.Error()
}
target.Results.TimedOut = err == context.DeadlineExceeded
coverage = parseCoverageFile(target, coverageFile)
target.Results.Duration += duration
if !core.PathExists(outputFile) {
if err == nil && target.NoTestOutput {
target.Results.NumTests += 1
target.Results.Passed += 1
numSucceeded++
} else if err == nil {
target.Results.NumTests++
target.Results.Failed++
target.Results.Failures = append(target.Results.Failures, core.TestFailure{
Name: "Missing results",
Stdout: string(out),
})
resultErr = fmt.Errorf("Test failed to produce output results file")
resultMsg = fmt.Sprintf("Test apparently succeeded but failed to produce %s. Output: %s", outputFile, string(out))
numFlakes++
} else {
target.Results.NumTests++
target.Results.Failed++
target.Results.Failures = append(target.Results.Failures, core.TestFailure{
Name: "Test failed with no results",
Stdout: string(out),
})
numFlakes++
resultErr = err
resultMsg = fmt.Sprintf("Test failed with no results. Output: %s", string(out))
}
} else {
results, err2 := parseTestResults(target, outputFile, false)
if err2 != nil {
resultErr = err2
resultMsg = fmt.Sprintf("Couldn't parse test output file: %s. Stdout: %s", err2, string(out))
numFlakes++
} else if err != nil && results.Failed == 0 {
// Add a failure result to the test so it shows up in the final aggregation.
target.Results.Failed = 1
target.Results.Failures = append(results.Failures, core.TestFailure{
Name: "Return value",
Type: fmt.Sprintf("%s", err),
Stdout: string(out),
})
numFlakes++
resultErr = err
resultMsg = fmt.Sprintf("Test returned nonzero but reported no errors: %s. Output: %s", err, string(out))
} else if err == nil && results.Failed != 0 {
resultErr = fmt.Errorf("Test returned 0 but still reported failures")
resultMsg = fmt.Sprintf("Test returned 0 but still reported failures. Stdout: %s", string(out))
numFlakes++
} else if results.Failed != 0 {
resultErr = fmt.Errorf("Tests failed")
resultMsg = fmt.Sprintf("Tests failed. Stdout: %s", string(out))
numFlakes++
} else {
numSucceeded++
if !state.ShowTestOutput {
// Save a bit of memory, if we're not printing results on success we will never use them again.
target.Results.Output = ""
}
}
}
}
if numSucceeded >= successesRequired {
target.Results.Failures = nil // Remove any failures, they don't count
target.Results.Failed = 0 // (they'll be picked up as flakes below)
if numSucceeded > 0 && numFlakes > 0 {
target.Results.Flakes = numFlakes
}
// Success, clean things up
if moveAndCacheOutputFiles(target.Results, coverage) {
logTestSuccess(state, tid, label, target.Results, coverage)
}
// Clean up the test directory.
if state.CleanWorkdirs {
if err := os.RemoveAll(target.TestDir()); err != nil {
log.Warning("Failed to remove test directory for %s: %s", target.Label, err)
}
}
} else {
state.LogTestResult(tid, label, core.TargetTestFailed, target.Results, coverage, resultErr, resultMsg)
}
}