func TestRetrieve(t *testing.T) {
target := core.NewBuildTarget(label)
target.AddOutput("testfile")
if !rpccache.Retrieve(target, []byte("test_key")) {
t.Error("Artifact expected and not found.")
}
}
func addTarget(state *core.BuildState, i int) *core.BuildTarget {
// Create and add a new target, with a parent and a dependency.
target := core.NewBuildTarget(label(i))
target.Command = "__FILEGROUP__" // Will mean it doesn't have to shell out to anything.
target.SetState(core.Active)
state.Graph.AddTarget(target)
if i <= size {
if i > 10 {
target.Flakiness = i // Stash this here, will be useful later.
target.PostBuildFunction = reflect.ValueOf(&postBuildFunc).Pointer()
}
if i < size/10 {
for j := 0; j < 10; j++ {
dep := label(i*10 + j)
log.Info("Adding dependency %s -> %s", target.Label, dep)
target.AddDependency(dep)
state.Graph.AddDependency(target.Label, dep)
}
} else {
// These are buildable now
state.AddPendingBuild(target.Label, false)
}
}
state.AddActiveTarget()
return target
}
func makeTarget(label string, deps ...string) *core.BuildTarget {
target := core.NewBuildTarget(core.ParseBuildLabel(label, ""))
for _, dep := range deps {
target.AddDependency(core.ParseBuildLabel(dep, ""))
}
return target
}
func getSubincludeFile(pkg *core.Package, labelStr string) string {
label := core.ParseBuildLabel(labelStr, pkg.Name)
if label.PackageName == pkg.Name {
return fmt.Sprintf("__Can't subinclude :%s in %s; can't subinclude local targets.", label.Name, pkg.Name)
}
pkgLabel := core.BuildLabel{PackageName: pkg.Name, Name: "all"}
target := core.State.Graph.Target(label)
if target == nil {
// Might not have been parsed yet. Check for that first.
if subincludePackage := core.State.Graph.Package(label.PackageName); subincludePackage == nil {
if deferParse(label, pkg) {
return pyDeferParse // Not an error, they'll just have to wait.
}
target = core.State.Graph.TargetOrDie(label) // Should be there now.
} else {
return fmt.Sprintf("__Failed to subinclude %s; package %s has no target by that name", label, label.PackageName)
}
} else if tmp := core.NewBuildTarget(pkgLabel); !tmp.CanSee(target) {
return fmt.Sprintf("__Can't subinclude %s from %s due to visibility constraints", label, pkg.Name)
} else if len(target.Outputs()) != 1 {
return fmt.Sprintf("__Can't subinclude %s, subinclude targets must have exactly one output", label)
} else if target.State() < core.Built {
if deferParse(label, pkg) {
return pyDeferParse // Again, they'll have to wait for this guy to build.
}
}
pkg.RegisterSubinclude(target.Label)
// Well if we made it to here it's actually ready to go, so tell them where to get it.
return path.Join(target.OutDir(), target.Outputs()[0])
}
func TestSomeMetrics(t *testing.T) {
m := initMetrics(url, verySlow, timeout, nil)
assert.Equal(t, 0, m.errors)
assert.Equal(t, 0, m.pushes)
m.record(core.NewBuildTarget(label), time.Millisecond)
m.stop()
assert.Equal(t, 1, m.errors, "Stop should push once more when there are metrics")
}
func newState(label string) (*core.BuildState, *core.BuildTarget) {
config, _ := core.ReadConfigFiles(nil)
state := core.NewBuildState(1, nil, 4, config)
target := core.NewBuildTarget(core.ParseBuildLabel(label, ""))
target.Command = fmt.Sprintf("echo 'output of %s' > $OUT", target.Label)
state.Graph.AddTarget(target)
return state, target
}
func TestClean(t *testing.T) {
target := core.NewBuildTarget(label)
rpccache.Clean(target)
filename := path.Join("src/cache/test_data", osName, "pkg/name/label_name")
if core.PathExists(filename) {
t.Errorf("File %s was not removed from cache.", filename)
}
}
func TestStore(t *testing.T) {
target := core.NewBuildTarget(label)
target.AddOutput("testfile2")
rpccache.Store(target, []byte("test_key"))
expectedPath := path.Join("src/cache/test_data", osName, "pkg/name", "label_name", "dGVzdF9rZXk", target.Outputs()[0])
if !core.PathExists(expectedPath) {
t.Errorf("Test file %s was not stored in cache.", expectedPath)
}
}
func newPyFilegroup(state *core.BuildState, label, filename string) *core.BuildTarget {
target := core.NewBuildTarget(core.ParseBuildLabel(label, ""))
target.AddSource(core.FileLabel{File: filename, Package: target.Label.PackageName})
target.AddOutput(filename)
target.AddLabel("py")
target.Command = "__FILEGROUP__" // magic
state.Graph.AddTarget(target)
return target
}
func TestExportedFunctions(t *testing.T) {
// For various reasons it's important that this is the only test that uses the global singleton.
config := core.DefaultConfiguration()
config.Metrics.PushGatewayURL = url
config.Metrics.PushFrequency = verySlow
InitFromConfig(config)
Record(core.NewBuildTarget(label), time.Millisecond)
Stop()
assert.Equal(t, 1, m.errors)
}
func TestPushAttempts(t *testing.T) {
m := initMetrics(url, 1, 1000, nil) // Fast push attempts
assert.Equal(t, 0, m.errors)
assert.Equal(t, 0, m.pushes)
m.record(core.NewBuildTarget(label), time.Millisecond)
time.Sleep(50 * time.Millisecond) // Not ideal but should be heaps of time for it to attempt pushes.
assert.Equal(t, maxErrors, m.errors)
assert.True(t, m.cancelled)
m.stop()
assert.Equal(t, maxErrors, m.errors, "Should not push again if it's hit the max errors")
}
func makeTarget(name string, command string, dep *core.BuildTarget) *core.BuildTarget {
target := core.NewBuildTarget(core.ParseBuildLabel(name, ""))
target.Command = command
target.AddOutput(target.Label.Name + ".py")
if dep != nil {
target.AddDependency(dep.Label)
// This is a bit awkward but I don't want to add a public interface just for a test.
graph := core.NewGraph()
graph.AddTarget(target)
graph.AddTarget(dep)
graph.AddDependency(target.Label, dep.Label)
}
return target
}
func TestGetLabels(t *testing.T) {
state := core.NewBuildState(10, nil, 2, core.DefaultConfiguration())
target1 := core.NewBuildTarget(core.ParseBuildLabel("//src/parse:target1", ""))
target2 := core.NewBuildTarget(core.ParseBuildLabel("//src/parse:target2", ""))
target3 := core.NewBuildTarget(core.ParseBuildLabel("//src/parse:target3", ""))
target1.AddLabel("go")
target2.AddLabel("py")
target3.AddLabel("cc")
target1.AddDependency(target2.Label)
target1.AddDependency(target3.Label)
target2.AddDependency(target3.Label)
state.Graph.AddTarget(target1)
state.Graph.AddTarget(target2)
state.Graph.AddTarget(target3)
state.Graph.AddDependency(target1.Label, target2.Label)
state.Graph.AddDependency(target1.Label, target3.Label)
state.Graph.AddDependency(target2.Label, target3.Label)
// Note labels always come out in sorted order.
assert.Equal(t, []string{"cc", "go", "py"}, getLabels(target1, "", core.Inactive))
assert.Equal(t, []string{"cc", "py"}, getLabels(target2, "", core.Inactive))
assert.Equal(t, []string{"cc"}, getLabels(target3, "", core.Inactive))
assert.Equal(t, []string{"y"}, getLabels(target1, "p", core.Inactive))
}
func init() {
osName = runtime.GOOS + "_" + runtime.GOARCH
label = core.NewBuildLabel("pkg/name", "label_name")
target = core.NewBuildTarget(label)
// Arbitrary large numbers so the cleaner never needs to run.
cache := server.NewCache("src/cache/test_data", 20*time.Hour, 100000, 100000000, 1000000000)
key, _ = ioutil.ReadFile("src/cache/test_data/testfile")
testServer := httptest.NewServer(server.BuildRouter(cache))
config := core.DefaultConfiguration()
config.Cache.HttpUrl = testServer.URL
config.Cache.HttpWriteable = true
httpcache = newHttpCache(config)
}
func TestStoreAndRetrieve(t *testing.T) {
target := core.NewBuildTarget(label)
target.AddOutput("testfile3")
rpccache.Store(target, []byte("test_key"))
// Remove the file so we can test retrieval correctly
outPath := path.Join(target.OutDir(), target.Outputs()[0])
if err := os.Remove(outPath); err != nil {
t.Errorf("Failed to remove artifact: %s", err)
}
if !rpccache.Retrieve(target, []byte("test_key")) {
t.Error("Artifact expected and not found.")
} else if !core.PathExists(outPath) {
t.Errorf("Artifact %s doesn't exist after alleged cache retrieval", outPath)
}
}
func makeTarget(g *core.BuildGraph, packageName string, labelName string, outputs []string) *core.BuildTarget {
l := core.ParseBuildLabel(fmt.Sprintf("//%s:%s", packageName, labelName), "")
t := core.NewBuildTarget(l)
p := g.Package(packageName)
if p == nil {
p = core.NewPackage(packageName)
g.AddPackage(p)
}
for _, out := range outputs {
t.AddOutput(out)
p.MustRegisterOutput(out, t)
}
p.Targets[labelName] = t
g.AddTarget(t)
return t
}
// addTarget adds a new build target to the graph.
// Separated from AddTarget to make it possible to test (since you can't mix cgo and go test).
func addTarget(pkgPtr uintptr, name, cmd, testCmd string, binary, test, needsTransitiveDeps,
outputIsComplete, containerise, noTestOutput, testOnly, stamp bool,
flakiness, buildTimeout, testTimeout int, buildingDescription string) *core.BuildTarget {
pkg := unsizep(pkgPtr)
target := core.NewBuildTarget(core.NewBuildLabel(pkg.Name, name))
target.IsBinary = binary
target.IsTest = test
target.NeedsTransitiveDependencies = needsTransitiveDeps
target.OutputIsComplete = outputIsComplete
target.Containerise = containerise
target.NoTestOutput = noTestOutput
target.TestOnly = testOnly
target.Flakiness = flakiness
target.BuildTimeout = time.Duration(buildTimeout) * time.Second
target.TestTimeout = time.Duration(testTimeout) * time.Second
target.Stamp = stamp
// Automatically label containerised tests.
if containerise {
target.AddLabel("container")
}
// Automatically label flaky tests.
if flakiness > 0 {
target.AddLabel("flaky")
}
if binary {
target.AddLabel("bin")
}
if buildingDescription != "" {
target.BuildingDescription = buildingDescription
}
target.Command = cmd
target.TestCommand = testCmd
if _, present := pkg.Targets[name]; present {
// NB. Not logged as an error because Python is now allowed to catch it.
// It will turn into an error later if the exception is not caught.
log.Notice("Duplicate build target in %s: %s", pkg.Name, name)
return nil
}
pkg.Targets[name] = target
if core.State.Graph.Package(pkg.Name) != nil {
// Package already added, so we're probably in a post-build function. Add target directly to graph now.
log.Debug("Adding new target %s directly to graph", target.Label)
core.State.Graph.AddTarget(target)
}
return target
}
func TestDisconnectAfterEnoughErrors(t *testing.T) {
// Need a separate cache for this so we don't interfere with the other tests.
s := startServer(7676, "", "", "")
c := buildClient(7676, "")
target := core.NewBuildTarget(label)
target.AddOutput("testfile4")
key := []byte("test_key")
c.Store(target, []byte("test_key"))
assert.True(t, c.Retrieve(target, key))
s.Stop()
// Now after we hit the max number of errors it should disconnect.
for i := 0; i < maxErrors; i++ {
assert.True(t, c.Connected)
assert.False(t, c.Retrieve(target, key))
}
assert.False(t, c.Connected)
}
func TestTargetStates(t *testing.T) {
m := initMetrics(url, verySlow, timeout, nil)
assert.Equal(t, 0, m.errors)
assert.Equal(t, 0, m.pushes)
target := core.NewBuildTarget(label)
m.record(target, time.Millisecond)
target.SetState(core.Cached)
m.record(target, time.Millisecond)
target.SetState(core.Built)
m.record(target, time.Millisecond)
target.Results.NumTests = 3
m.record(target, time.Millisecond)
target.Results.Failed = 1
m.record(target, time.Millisecond)
target.Results.Cached = true
m.record(target, time.Millisecond)
m.stop()
assert.Equal(t, 1, m.errors)
}
func TestGetSubincludeFile(t *testing.T) {
assertError := func(t *testing.T, ret, msg string) { assert.True(t, strings.HasPrefix(ret, "__"), msg) }
state := core.NewBuildState(10, nil, 2, core.DefaultConfiguration())
pkg := core.NewPackage("src/parse")
pkg2 := core.NewPackage("src/core")
assert.Equal(t, pyDeferParse, getSubincludeFile(pkg, "//src/core:target"), "Package not loaded yet, should defer")
assertError(t, getSubincludeFile(pkg, "//src/parse:target"), "Should produce an error on attempts for local subincludes.")
assertError(t, getSubincludeFile(pkg, ":target"), "Should produce an error on attempts for local subincludes.")
state.Graph.AddPackage(pkg)
state.Graph.AddPackage(pkg2)
assertError(t, getSubincludeFile(pkg, "//src/core:target"), "Produces an error, target does not exist in package.")
target := core.NewBuildTarget(core.ParseBuildLabel("//src/core:target", ""))
state.Graph.AddTarget(target)
assertError(t, getSubincludeFile(pkg, "//src/core:target"), "Errors, target is not visible to subincluding package.")
target.Visibility = []core.BuildLabel{core.ParseBuildLabel("//src/parse:all", "")}
assertError(t, getSubincludeFile(pkg, "//src/core:target"), "Errors, target doesn't have any outputs to include.")
target.AddOutput("test.py")
assert.Equal(t, pyDeferParse, getSubincludeFile(pkg, "//src/core:target"), "Target isn't built yet, so still deferred")
target.SetState(core.Built)
assert.Equal(t, "plz-out/gen/src/core/test.py", getSubincludeFile(pkg, "//src/core:target"), "Success at last")
}
func makeTarget(label string) *core.BuildTarget {
return core.NewBuildTarget(core.ParseBuildLabel(label, ""))
}