func addJSONTarget(graph *core.BuildGraph, ret *JSONGraph, label core.BuildLabel, done map[core.BuildLabel]struct{}) {
if _, present := done[label]; present {
return
}
done[label] = struct{}{}
if label.IsAllTargets() {
pkg := graph.PackageOrDie(label.PackageName)
for _, target := range pkg.Targets {
addJSONTarget(graph, ret, target.Label, done)
}
return
}
target := graph.TargetOrDie(label)
if _, present := ret.Packages[label.PackageName]; present {
ret.Packages[label.PackageName].Targets[label.Name] = makeJSONTarget(graph, target)
} else {
ret.Packages[label.PackageName] = JSONPackage{
Targets: map[string]JSONTarget{
label.Name: makeJSONTarget(graph, target),
},
}
}
for _, dep := range target.Dependencies() {
addJSONTarget(graph, ret, dep.Label, done)
}
}
// Run implements the running part of 'plz run'.
func Run(graph *core.BuildGraph, label core.BuildLabel, args []string) {
target := graph.TargetOrDie(label)
if !target.IsBinary {
log.Fatalf("Target %s cannot be run; it's not marked as binary", label)
}
// ReplaceSequences always quotes stuff in case it contains spaces or special characters,
// that works fine if we interpret it as a shell but not to pass it as an argument here.
cmd := strings.Trim(build.ReplaceSequences(target, fmt.Sprintf("$(out_exe %s)", target.Label)), "\"")
// Handle targets where $(exe ...) returns something nontrivial
splitCmd := strings.Split(cmd, " ")
if !strings.Contains(splitCmd[0], "/") {
// Probably it's a java -jar, we need an absolute path to it.
cmd, err := exec.LookPath(splitCmd[0])
if err != nil {
log.Fatalf("Can't find binary %s", splitCmd[0])
}
splitCmd[0] = cmd
}
args = append(splitCmd, args...)
log.Info("Running target %s...", strings.Join(args, " "))
output.SetWindowTitle("plz run: " + strings.Join(args, " "))
if err := syscall.Exec(splitCmd[0], args, os.Environ()); err != nil {
log.Fatalf("Error running command %s: %s", strings.Join(args, " "), err)
}
}
func addDeps(graph *core.BuildGraph, pkg *core.Package) {
for _, target := range pkg.Targets {
for _, dep := range target.DeclaredDependencies() {
graph.AddDependency(target.Label, dep)
}
}
}
// Set dependency pointers on all contents of the graph.
// Has to be done after to test cycles etc.
func updateDependencies(graph *core.BuildGraph) {
for _, target := range graph.AllTargets() {
for _, dep := range target.DeclaredDependencies() {
graph.AddDependency(target.Label, dep)
}
}
}
// Calculate the hash of all sources of this rule
func sourceHash(graph *core.BuildGraph, target *core.BuildTarget) ([]byte, error) {
h := sha1.New()
for source := range core.IterSources(graph, target) {
result, err := pathHash(source.Src, false)
if err != nil {
return nil, err
}
h.Write(result)
}
for _, tool := range target.Tools {
if label := tool.Label(); label != nil {
// Note that really it would be more correct to hash the outputs of these rules
// in the same way we calculate a hash of sources for the rule, but that is
// impractical for some cases (notably npm) where tools can be very large.
// Instead we assume calculating the target hash is sufficient.
h.Write(mustTargetHash(core.State, graph.TargetOrDie(*label)))
} else {
result, err := pathHash(tool.FullPaths(graph)[0], false)
if err != nil {
return nil, err
}
h.Write(result)
}
}
return h.Sum(nil), nil
}
// QueryTargetOutputs prints all output files for a set of targets.
func QueryTargetOutputs(graph *core.BuildGraph, labels []core.BuildLabel) {
for _, label := range labels {
target := graph.TargetOrDie(label)
for _, out := range target.Outputs() {
fmt.Printf("%s\n", path.Join(target.OutDir(), out))
}
}
}
// QueryTargetInputs prints all inputs for a single target.
func QueryTargetInputs(graph *core.BuildGraph, labels []core.BuildLabel) {
inputPaths := map[string]bool{}
for _, label := range labels {
for sourcePath := range core.IterInputPaths(graph, graph.TargetOrDie(label)) {
inputPaths[sourcePath] = true
}
}
for path := range inputPaths {
fmt.Printf("%s\n", path)
}
}
func querySomePath1(graph *core.BuildGraph, target1 *core.BuildTarget, label2 core.BuildLabel, print bool) bool {
// Now we do the same for label2.
if label2.IsAllTargets() {
for _, target2 := range graph.PackageOrDie(label2.PackageName).Targets {
if querySomePath2(graph, target1, target2, false) {
return true
}
}
return false
}
return querySomePath2(graph, target1, graph.TargetOrDie(label2), print)
}
func filesToLabelMap(graph *core.BuildGraph) map[string]*core.BuildLabel {
packageMap := make(map[string]*core.BuildLabel)
for _, pkg := range graph.PackageMap() {
for _, target := range pkg.Outputs {
for _, output := range target.Outputs() {
artifactPath := path.Join(target.OutDir(), output)
packageMap[artifactPath] = &target.Label
}
}
}
return packageMap
}
// ReverseDeps For each input label, finds all targets which depend upon it.
func ReverseDeps(graph *core.BuildGraph, labels []core.BuildLabel) {
uniqueTargets := make(map[core.BuildLabel]struct{})
for _, label := range labels {
for _, child := range graph.PackageOrDie(label.PackageName).AllChildren(graph.TargetOrDie(label)) {
for _, target := range graph.ReverseDependencies(child) {
if parent := target.Parent(graph); parent != nil {
uniqueTargets[parent.Label] = struct{}{}
} else {
uniqueTargets[target.Label] = struct{}{}
}
}
}
}
// Check for anything subincluding this guy too
for _, pkg := range graph.PackageMap() {
for _, label := range labels {
if pkg.HasSubinclude(label) {
uniqueTargets[core.BuildLabel{PackageName: pkg.Name, Name: "all"}] = struct{}{}
}
}
}
targets := make(core.BuildLabels, 0, len(uniqueTargets))
for target := range uniqueTargets {
targets = append(targets, target)
}
sort.Sort(targets)
for _, target := range targets {
fmt.Printf("%s\n", target)
}
}
func makeJSONGraph(graph *core.BuildGraph, targets []core.BuildLabel) *JSONGraph {
ret := JSONGraph{Packages: map[string]JSONPackage{}}
if len(targets) == 0 {
for name, pkg := range graph.PackageMap() {
ret.Packages[name] = makeJSONPackage(graph, pkg)
}
} else {
done := map[core.BuildLabel]struct{}{}
for _, target := range targets {
addJSONTarget(graph, &ret, target, done)
}
}
return &ret
}
// QuerySomePath finds and returns a path between two targets.
// Useful for a "why on earth do I depend on this thing" type query.
func QuerySomePath(graph *core.BuildGraph, label1 core.BuildLabel, label2 core.BuildLabel) {
// Awkwardly either target can be :all. This is an extremely useful idiom though so despite
// trickiness is worth supporting.
// Of course this calculation is also quadratic but it's not very obvious how to avoid that.
if label1.IsAllTargets() {
for _, target := range graph.PackageOrDie(label1.PackageName).Targets {
if querySomePath1(graph, target, label2, false) {
return
}
}
fmt.Printf("Couldn't find any dependency path between %s and %s\n", label1, label2)
} else {
querySomePath1(graph, graph.TargetOrDie(label1), label2, true)
}
}
// This is just a simple DFS through the graph.
func printSomePath(graph *core.BuildGraph, target1, target2 *core.BuildTarget) bool {
if target1 == target2 {
fmt.Printf("Found path:\n %s\n", target1.Label)
return true
}
for _, target := range graph.ReverseDependencies(target2) {
if printSomePath(graph, target1, target) {
if target2.Parent(graph) != target {
fmt.Printf(" %s\n", target2.Label)
}
return true
}
}
return false
}
// Queries a set of possible completions for some build labels.
// If 'binary' is true it will complete only targets that are runnable binaries (but not tests).
// If 'test' is true it will similarly complete only targets that are tests.
func QueryCompletions(graph *core.BuildGraph, labels []core.BuildLabel, binary, test bool) {
for _, label := range labels {
count := 0
for _, target := range graph.PackageOrDie(label.PackageName).Targets {
if (binary && (!target.IsBinary || target.IsTest)) || (test && !target.IsTest) {
continue
}
if !strings.HasPrefix(target.Label.Name, "_") {
fmt.Printf("%s\n", target.Label)
count++
}
}
if !binary && count > 1 {
fmt.Printf("//%s:all\n", label.PackageName)
}
}
}
// Prints all targets in the build graph that are marked to be built but not built yet.
func unbuiltTargetsMessage(graph *core.BuildGraph) string {
msg := ""
for _, target := range graph.AllTargets() {
if target.State() == core.Active {
if graph.AllDepsBuilt(target) {
msg += fmt.Sprintf(" %s (waiting for deps to build)\n", target.Label)
} else {
msg += fmt.Sprintf(" %s\n", target.Label)
}
} else if target.State() == core.Pending {
msg += fmt.Sprintf(" %s (pending build)\n", target.Label)
}
}
if msg != "" {
return "\nThe following targets have not yet built:\n" + msg
}
return ""
}
func handleAffectedTargets(graph *core.BuildGraph, affectedTargets <-chan *core.BuildTarget, done chan<- bool, include, exclude []string, tests, transitive bool) {
seenTargets := map[*core.BuildTarget]bool{}
var inner func(*core.BuildTarget)
inner = func(target *core.BuildTarget) {
if !seenTargets[target] {
seenTargets[target] = true
if transitive {
for _, revdep := range graph.ReverseDependencies(target) {
inner(revdep)
}
}
if (!tests || target.IsTest) && target.ShouldInclude(include, exclude) {
fmt.Printf("%s\n", target.Label)
}
}
}
for target := range affectedTargets {
inner(target)
}
done <- true
}
// Write test results out to a file in xUnit format. Dies on any errors.
func WriteResultsToFileOrDie(graph *core.BuildGraph, filename string) {
if err := os.MkdirAll(path.Dir(filename), core.DirPermissions); err != nil {
log.Fatalf("Failed to create directory for test output")
}
results := JUnitXMLTestResults{}
results.XMLName.Local = "testsuites"
for _, target := range graph.AllTargets() {
if target.Results.NumTests > 0 {
suite := JUnitXMLTestSuite{
Name: target.Label.String(),
Failures: target.Results.Failed,
Tests: target.Results.NumTests,
}
for _, pass := range target.Results.Passes {
suite.TestCases = append(suite.TestCases, JUnitXMLTest{Name: pass})
}
for _, fail := range target.Results.Failures {
suite.TestCases = append(suite.TestCases, JUnitXMLTest{
Name: fail.Name,
Type: fail.Type,
Stdout: fail.Stdout,
Stderr: fail.Stderr,
Error: &JUnitXMLFailure{
Type: fail.Type,
Traceback: fail.Traceback,
},
})
}
results.TestSuites = append(results.TestSuites, suite)
}
}
if b, err := xml.MarshalIndent(results, "", " "); err != nil {
log.Fatalf("Failed to serialise XML: %s", err)
} else if err = ioutil.WriteFile(filename, b, 0644); err != nil {
log.Fatalf("Failed to write XML to %s: %s", filename, err)
}
}
// QueryAffectedTargets walks over the build graph and identifies all targets that have a transitive
// dependency on the given set of files.
// Targets are filtered by given include / exclude labels and if 'tests' is true only
// test targets will be returned.
func QueryAffectedTargets(graph *core.BuildGraph, files, include, exclude []string, tests, transitive bool) {
affectedTargets := make(chan *core.BuildTarget, 100)
done := make(chan bool)
filePaths := map[string]bool{}
for _, file := range files {
filePaths[file] = true
}
// Check all the targets to see if any own one of these files
go func() {
for _, target := range graph.AllTargets() {
for _, source := range target.AllSourcePaths(graph) {
if _, present := filePaths[source]; present {
affectedTargets <- target
break
}
}
}
done <- true
}()
// Check all the packages to see if any are defined by these files.
// This is pretty pessimistic, we have to just assume the whole package is invalidated.
// A better approach involves using plz query graph and plz_diff_graphs - see that tool
// for more explanation.
go func() {
invalidatePackage := func(pkg *core.Package) {
for _, target := range pkg.Targets {
affectedTargets <- target
}
}
for _, pkg := range graph.PackageMap() {
if _, present := filePaths[pkg.Filename]; present {
invalidatePackage(pkg)
} else {
for _, subinclude := range pkg.Subincludes {
for _, source := range graph.TargetOrDie(subinclude).AllSourcePaths(graph) {
if _, present := filePaths[source]; present {
invalidatePackage(pkg)
break
}
}
}
}
}
done <- true
}()
go handleAffectedTargets(graph, affectedTargets, done, include, exclude, tests, transitive)
<-done
<-done
close(affectedTargets)
<-done
}
func makeTarget(g *core.BuildGraph, packageName string, labelName string, outputs []string) *core.BuildTarget {
l := core.ParseBuildLabel(fmt.Sprintf("//%s:%s", packageName, labelName), "")
t := core.NewBuildTarget(l)
p := g.Package(packageName)
if p == nil {
p = core.NewPackage(packageName)
g.AddPackage(p)
}
for _, out := range outputs {
t.AddOutput(out)
p.MustRegisterOutput(out, t)
}
p.Targets[labelName] = t
g.AddTarget(t)
return t
}
// QueryPrint produces a Python call which would (hopefully) regenerate the same build rule if run.
// This is of course not ideal since they were almost certainly created as a java_library
// or some similar wrapper rule, but we've lost that information by now.
func QueryPrint(graph *core.BuildGraph, labels []core.BuildLabel) {
for _, label := range labels {
target := graph.TargetOrDie(label)
fmt.Printf("%s:\n", label)
if target.IsFilegroup() {
fmt.Printf(" filegroup(\n")
} else {
fmt.Printf(" build_rule(\n")
}
fmt.Printf(" name = '%s'\n", target.Label.Name)
if len(target.Sources) > 0 {
fmt.Printf(" srcs = [\n")
for _, src := range target.Sources {
fmt.Printf(" '%s',\n", src)
}
fmt.Printf(" ],\n")
} else if target.NamedSources != nil {
fmt.Printf(" srcs = {\n")
for name, srcs := range target.NamedSources {
fmt.Printf(" '%s': [\n", name)
for _, src := range srcs {
fmt.Printf(" '%s'\n", src)
}
fmt.Printf(" ],\n")
}
fmt.Printf(" },\n")
}
if len(target.DeclaredOutputs()) > 0 && !target.IsFilegroup() {
fmt.Printf(" outs = [\n")
for _, out := range target.DeclaredOutputs() {
fmt.Printf(" '%s',\n", out)
}
fmt.Printf(" ],\n")
}
stringList("optional_outs", target.OptionalOutputs)
pythonDict(target.Commands, "cmd")
if !target.IsFilegroup() {
fmt.Printf(" cmd = '%s'\n", target.Command)
}
pythonDict(target.TestCommands, "test_cmd")
if target.TestCommand != "" {
fmt.Printf(" test_cmd = '%s'\n", target.TestCommand)
}
pythonBool("binary", target.IsBinary)
pythonBool("test", target.IsTest)
pythonBool("needs_transitive_deps", target.NeedsTransitiveDependencies)
if !target.IsFilegroup() {
pythonBool("output_is_complete", target.OutputIsComplete)
if target.BuildingDescription != core.DefaultBuildingDescription {
fmt.Printf(" building_description = '%s',\n", target.BuildingDescription)
}
}
pythonBool("stamp", target.Stamp)
if target.ContainerSettings != nil {
fmt.Printf(" container = {\n")
fmt.Printf(" 'docker_image': '%s',\n", target.ContainerSettings.DockerImage)
fmt.Printf(" 'docker_user': '******',\n", target.ContainerSettings.DockerUser)
fmt.Printf(" 'docker_run_args': '%s',\n", target.ContainerSettings.DockerRunArgs)
} else {
pythonBool("container", target.Containerise)
}
pythonBool("no_test_output", target.NoTestOutput)
pythonBool("test_only", target.TestOnly)
labelList("deps", excludeLabels(target.DeclaredDependencies(), target.ExportedDependencies(), sourceLabels(target)), target)
labelList("exported_deps", target.ExportedDependencies(), target)
if len(target.Tools) > 0 {
fmt.Printf(" tools = [\n")
for _, tool := range target.Tools {
fmt.Printf(" '%s',\n", tool)
}
fmt.Printf(" ],\n")
}
if len(target.Data) > 0 {
fmt.Printf(" data = [\n")
for _, datum := range target.Data {
fmt.Printf(" '%s',\n", datum)
}
fmt.Printf(" ],\n")
}
stringList("labels", excludeStrings(target.Labels, target.Requires))
stringList("hashes", target.Hashes)
stringList("licences", target.Licences)
stringList("test_outputs", target.TestOutputs)
stringList("requires", target.Requires)
if len(target.Provides) > 0 {
fmt.Printf(" provides = {\n")
for k, v := range target.Provides {
if v.PackageName == target.Label.PackageName {
fmt.Printf(" '%s': ':%s',\n", k, v.Name)
} else {
fmt.Printf(" '%s': '%s',\n", k, v)
}
}
fmt.Printf(" },\n")
}
if target.Flakiness > 0 {
fmt.Printf(" flaky = %d,\n", target.Flakiness)
}
if target.BuildTimeout > 0 {
fmt.Printf(" timeout = %d,\n", target.BuildTimeout)
}
if target.TestTimeout > 0 {
fmt.Printf(" test_timeout = %d,\n", target.TestTimeout)
}
if len(target.Visibility) > 0 {
fmt.Printf(" visibility = [\n")
for _, vis := range target.Visibility {
if vis.PackageName == "" && vis.IsAllSubpackages() {
fmt.Printf(" 'PUBLIC',\n")
} else {
fmt.Printf(" '%s',\n", vis)
}
}
fmt.Printf(" ],\n")
}
if target.PreBuildFunction != 0 {
fmt.Printf(" pre_build = <python ref>,\n") // Don't have any sensible way of printing this.
}
if target.PostBuildFunction != 0 {
fmt.Printf(" post_build = <python ref>,\n") // Don't have any sensible way of printing this.
}
fmt.Printf(" )\n\n")
}
}
