// importPathsNoDotExpansion returns the import paths to use for the given
// command line, but it does no ... expansion.
func importPathsNoDotExpansion(ctx Context, cwd string, args []string) []string {
srcdir, _ := filepath.Rel(filepath.Join(ctx.Projectdir(), "src"), cwd)
debug.Debugf("%s %s", cwd, srcdir)
if srcdir == ".." {
srcdir = "."
}
if len(args) == 0 {
args = []string{"..."}
}
var out []string
for _, a := range args {
// Arguments are supposed to be import paths, but
// as a courtesy to Windows developers, rewrite \ to /
// in command-line arguments. Handles .\... and so on.
if filepath.Separator == '\\' {
a = strings.Replace(a, `\`, `/`, -1)
}
if a == "all" || a == "std" {
pkgs, err := ctx.AllPackages(a)
if err != nil {
fatalf("could not load all packages: %v", err)
}
out = append(out, pkgs...)
continue
}
a = path.Join(srcdir, path.Clean(a))
out = append(out, a)
}
return out
}
func runOut(output io.Writer, dir string, env []string, command string, args ...string) error {
cmd := exec.Command(command, args...)
cmd.Dir = dir
cmd.Stdout = output
cmd.Stderr = os.Stderr
cmd.Env = mergeEnvLists(env, envForDir(cmd.Dir))
debug.Debugf("cd %s; %s", cmd.Dir, cmd.Args)
err := cmd.Run()
return err
}
// BuildPackages produces a tree of *Actions that can be executed to build
// a *Package.
// BuildPackages walks the tree of *Packages and returns a corresponding
// tree of *Actions representing the steps required to build *Package
// and any of its dependencies
func BuildPackages(pkgs ...*Package) (*Action, error) {
if len(pkgs) < 1 {
return nil, errors.New("no packages supplied")
}
targets := make(map[string]*Action) // maps package importpath to build action
names := func(pkgs []*Package) []string {
var names []string
for _, pkg := range pkgs {
names = append(names, pkg.ImportPath)
}
return names
}
// create top level build action to unify all packages
t0 := time.Now()
build := Action{
Name: fmt.Sprintf("build: %s", strings.Join(names(pkgs), ",")),
Run: func() error {
debug.Debugf("build duration: %v %v", time.Since(t0), pkgs[0].Statistics.String())
return nil
},
}
for _, pkg := range pkgs {
if len(pkg.GoFiles)+len(pkg.CgoFiles) == 0 {
debug.Debugf("skipping %v: no go files", pkg.ImportPath)
continue
}
a, err := BuildPackage(targets, pkg)
if err != nil {
return nil, err
}
if a == nil {
// nothing to do
continue
}
build.Deps = append(build.Deps, a)
}
return &build, nil
}
// TestFlags appends "-test." for flags that are passed to the test binary.
func TestFlags(testArgs []string) []string {
debug.Debugf("TestFlags: args: %s", testArgs)
var targs []string
for _, arg := range testArgs {
var nArg, nVal, fArg string
fArg = arg
if !strings.Contains(arg, "-test.") {
nArg = strings.TrimPrefix(arg, "-")
if strings.Contains(nArg, "=") {
nArgVal := strings.Split(nArg, "=")
nArg, nVal = nArgVal[0], nArgVal[1]
}
if val, ok := testFlagDefn[nArg]; ok {
// Special handling for -q, needs to be -test.v when passed to the test
if nArg == "q" {
nArg = "v"
}
if val.passToTest || val.passToAll {
fArg = "-test." + nArg
if val.boolVar {
// boolean variables can be either -bool, or -bool=true
// some code, see issue 605, expects the latter form, so
// when present, expand boolean args to their canonical
// form.
nVal = "true"
}
if nVal != "" {
fArg = fArg + "=" + nVal
}
}
}
}
targs = append(targs, fArg)
}
debug.Debugf("testFlags: targs: %s", targs)
return targs
}
func matchPackages(c *Context, pattern string) ([]string, error) {
debug.Debugf("matchPackages: %v", pattern)
match := func(string) bool { return true }
treeCanMatch := func(string) bool { return true }
if pattern != "all" && pattern != "std" {
match = matchPattern(pattern)
treeCanMatch = treeCanMatchPattern(pattern)
}
var pkgs []string
src := filepath.Join(c.Projectdir(), "src") + string(filepath.Separator)
err := filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
if err != nil || !fi.IsDir() || path == src {
return nil
}
// Avoid .foo, _foo, and testdata directory trees.
elem := fi.Name()
if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
return filepath.SkipDir
}
name := filepath.ToSlash(path[len(src):])
if pattern == "std" && strings.Contains(name, ".") {
return filepath.SkipDir
}
if !treeCanMatch(name) {
return filepath.SkipDir
}
if !match(name) {
return nil
}
_, err = c.importers[1].Import(name)
switch err.(type) {
case nil:
pkgs = append(pkgs, name)
return nil
case *importer.NoGoError:
return nil // skip
default:
return err
}
})
return pkgs, err
}
// loadTestFuncs returns the testFuncs describing the tests that will be run.
func loadTestFuncs(ptest *importer.Package) (*testFuncs, error) {
t := &testFuncs{
Package: ptest,
}
debug.Debugf("loadTestFuncs: %v, %v", ptest.TestGoFiles, ptest.XTestGoFiles)
for _, file := range ptest.TestGoFiles {
if err := t.load(filepath.Join(ptest.Dir, file), "_test", &t.ImportTest, &t.NeedTest); err != nil {
return nil, err
}
}
for _, file := range ptest.XTestGoFiles {
if err := t.load(filepath.Join(ptest.Dir, file), "_xtest", &t.ImportXtest, &t.NeedXtest); err != nil {
return nil, err
}
}
return t, nil
}
// NewContext creates a gb.Context for the project root.
func NewContext(projectroot string, options ...func(*gb.Context) error) (*gb.Context, error) {
if projectroot == "" {
return nil, errors.New("project root is blank")
}
root, err := FindProjectroot(projectroot)
if err != nil {
return nil, errors.Wrap(err, "could not locate project root")
}
project := gb.NewProject(root,
gb.SourceDir(filepath.Join(root, "src")),
gb.SourceDir(filepath.Join(root, "vendor", "src")),
)
debug.Debugf("project root %q", project.Projectdir())
return project.NewContext(options...)
}
// RunCommand detects the project root, parses flags and runs the Command.
func RunCommand(fs *flag.FlagSet, cmd *Command, projectroot, goroot string, args []string) error {
if cmd.AddFlags != nil {
cmd.AddFlags(fs)
}
if err := fs.Parse(args); err != nil {
fs.Usage()
os.Exit(1)
}
args = fs.Args() // reset to the remaining arguments
ctx, err := NewContext(projectroot, gb.GcToolchain())
if err != nil {
return errors.Wrap(err, "unable to construct context")
}
defer ctx.Destroy()
debug.Debugf("args: %v", args)
return cmd.Run(ctx, args)
}
// TestPackages produces a graph of Actions that when executed build
// and test the supplied packages.
func TestPackages(flags []string, pkgs ...*gb.Package) (*gb.Action, error) {
if len(pkgs) < 1 {
return nil, errors.New("no test packages provided")
}
targets := make(map[string]*gb.Action) // maps package import paths to their test run action
names := func(pkgs []*gb.Package) []string {
var names []string
for _, pkg := range pkgs {
names = append(names, pkg.ImportPath)
}
return names
}
// create top level test action to root all test actions
t0 := time.Now()
test := gb.Action{
Name: fmt.Sprintf("test: %s", strings.Join(names(pkgs), ",")),
Run: func() error {
debug.Debugf("test duration: %v %v", time.Since(t0), pkgs[0].Statistics.String())
return nil
},
}
for _, pkg := range pkgs {
a, err := TestPackage(targets, pkg, flags)
if err != nil {
return nil, err
}
if a == nil {
// nothing to do ?? not even a test action ?
continue
}
test.Deps = append(test.Deps, a)
}
return &test, nil
}
func main() {
args := os.Args
if len(args) < 2 || args[1] == "-h" {
fs.Usage() // usage calles exit(2)
}
name := args[1]
if name == "help" {
help(args[2:])
exit(0)
}
command, ok := commands[name]
if (command != nil && !command.Runnable()) || !ok {
plugin, err := lookupPlugin(name)
if err != nil {
fmt.Fprintf(os.Stderr, "FATAL: unknown command %q\n", name)
fs.Usage() // usage calles exit(2)
}
command = &cmd.Command{
Run: func(ctx *gb.Context, args []string) error {
args = append([]string{plugin}, args...)
env := cmd.MergeEnv(os.Environ(), map[string]string{
"GB_PROJECT_DIR": ctx.Projectdir(),
})
cmd := exec.Cmd{
Path: plugin,
Args: args,
Env: env,
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
return cmd.Run()
},
// plugin should not interpret arguments
SkipParseArgs: true,
}
}
// add extra flags if necessary
if command.AddFlags != nil {
command.AddFlags(fs)
}
var err error
if command.FlagParse != nil {
err = command.FlagParse(fs, args)
} else {
err = fs.Parse(args[2:])
}
if err != nil {
fatalf("could not parse flags: %v", err)
}
args = fs.Args() // reset args to the leftovers from fs.Parse
debug.Debugf("args: %v", args)
if command == commands["plugin"] {
args = append([]string{name}, args...)
}
cwd, err := filepath.Abs(cwd) // if cwd was passed in via -R, make sure it is absolute
if err != nil {
fatalf("could not make project root absolute: %v", err)
}
ctx, err := cmd.NewContext(
cwd, // project root
gb.GcToolchain(),
gb.Gcflags(gcflags...),
gb.Ldflags(ldflags...),
gb.Tags(buildtags...),
func(c *gb.Context) error {
if !race {
return nil
}
// check this is a supported platform
if runtime.GOARCH != "amd64" {
fatalf("race detector not supported on %s/%s", runtime.GOOS, runtime.GOARCH)
}
switch runtime.GOOS {
case "linux", "windows", "darwin", "freebsd":
// supported
default:
fatalf("race detector not supported on %s/%s", runtime.GOOS, runtime.GOARCH)
}
// check the race runtime is built
_, err := os.Stat(filepath.Join(runtime.GOROOT(), "pkg", fmt.Sprintf("%s_%s_race", runtime.GOOS, runtime.GOARCH), "runtime.a"))
if os.IsNotExist(err) || err != nil {
fatalf("go installation at %s is missing race support. See https://getgb.io/faq/#missing-race-support", runtime.GOROOT())
}
return gb.WithRace(c)
},
)
if err != nil {
fatalf("unable to construct context: %v", err)
}
if !command.SkipParseArgs {
args = importPaths(ctx, cwd, args)
}
debug.Debugf("args: %v", args)
if destroyContext {
atExit = append(atExit, ctx.Destroy)
}
if err := command.Run(ctx, args); err != nil {
fatalf("command %q failed: %v", name, err)
}
exit(0)
}
func main() {
fatalf := func(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
os.Exit(1)
}
args := os.Args[1:]
switch {
case len(args) < 1, args[0] == "-h", args[0] == "-help":
printUsage(os.Stdout)
os.Exit(0)
case args[0] == "help":
help(args[1:])
return
case projectroot == "":
fatalf("don't run this binary directly, it is meant to be run as 'gb vendor ...'")
default:
}
root, err := cmd.FindProjectroot(projectroot)
if err != nil {
fatalf("could not locate project root: %v", err)
}
project := gb.NewProject(root,
gb.SourceDir(filepath.Join(root, "src")),
gb.SourceDir(filepath.Join(root, "vendor", "src")),
)
debug.Debugf("project root %q", project.Projectdir())
for _, command := range commands {
if command.Name == args[0] && command.Runnable() {
// add extra flags if necessary
if command.AddFlags != nil {
command.AddFlags(fs)
}
if command.FlagParse != nil {
err = command.FlagParse(fs, args)
} else {
err = fs.Parse(args[1:])
}
if err != nil {
fatalf("could not parse flags: %v", err)
}
args = fs.Args() // reset args to the leftovers from fs.Parse
debug.Debugf("args: %v", args)
ctx, err := project.NewContext(
gb.GcToolchain(),
)
if err != nil {
fatalf("unable to construct context: %v", err)
}
defer ctx.Destroy()
if err := command.Run(ctx, args); err != nil {
fatalf("command %q failed: %v", command.Name, err)
}
return
}
}
fatalf("unknown command %q ", args[0])
}
// Destroy removes the temporary working files of this context.
func (c *Context) Destroy() error {
debug.Debugf("removing work directory: %v", c.workdir)
return os.RemoveAll(c.workdir)
}
// isStale returns true if the source pkg is considered to be stale with
// respect to its installed version.
func isStale(pkg *Package) bool {
switch pkg.ImportPath {
case "C", "unsafe":
// synthetic packages are never stale
return false
}
if !pkg.Standard && pkg.Force {
return true
}
// tests are always stale, they are never installed
if pkg.TestScope {
return true
}
// Package is stale if completely unbuilt.
var built time.Time
if fi, err := os.Stat(pkgpath(pkg)); err == nil {
built = fi.ModTime()
}
if built.IsZero() {
debug.Debugf("%s is missing", pkgpath(pkg))
return true
}
olderThan := func(file string) bool {
fi, err := os.Stat(file)
return err != nil || fi.ModTime().After(built)
}
newerThan := func(file string) bool {
fi, err := os.Stat(file)
return err != nil || fi.ModTime().Before(built)
}
// As a courtesy to developers installing new versions of the compiler
// frequently, define that packages are stale if they are
// older than the compiler, and commands if they are older than
// the linker. This heuristic will not work if the binaries are
// back-dated, as some binary distributions may do, but it does handle
// a very common case.
if !pkg.Standard {
if olderThan(pkg.tc.compiler()) {
debug.Debugf("%s is older than %s", pkgpath(pkg), pkg.tc.compiler())
return true
}
if pkg.isMain() && olderThan(pkg.tc.linker()) {
debug.Debugf("%s is older than %s", pkgpath(pkg), pkg.tc.compiler())
return true
}
}
if pkg.Standard && !pkg.isCrossCompile() {
// if this is a standard lib package, and we are not cross compiling
// then assume the package is up to date. This also works around
// golang/go#13769.
return false
}
// Package is stale if a dependency is newer.
for _, p := range pkg.Imports {
if p.ImportPath == "C" || p.ImportPath == "unsafe" {
continue // ignore stale imports of synthetic packages
}
if olderThan(pkgpath(p)) {
debug.Debugf("%s is older than %s", pkgpath(pkg), pkgpath(p))
return true
}
}
// if the main package is up to date but _newer_ than the binary (which
// could have been removed), then consider it stale.
if pkg.isMain() && newerThan(pkg.Binfile()) {
debug.Debugf("%s is newer than %s", pkgpath(pkg), pkg.Binfile())
return true
}
srcs := stringList(pkg.GoFiles, pkg.CFiles, pkg.CXXFiles, pkg.MFiles, pkg.HFiles, pkg.SFiles, pkg.CgoFiles, pkg.SysoFiles, pkg.SwigFiles, pkg.SwigCXXFiles)
for _, src := range srcs {
if olderThan(filepath.Join(pkg.Dir, src)) {
debug.Debugf("%s is older than %s", pkgpath(pkg), filepath.Join(pkg.Dir, src))
return true
}
}
return false
}
// TestPackage returns an Action representing the steps required to build
// and test this Package.
func TestPackage(targets map[string]*gb.Action, pkg *gb.Package, flags []string) (*gb.Action, error) {
debug.Debugf("TestPackage: %s, flags: %s", pkg.ImportPath, flags)
var gofiles []string
gofiles = append(gofiles, pkg.GoFiles...)
gofiles = append(gofiles, pkg.TestGoFiles...)
var cgofiles []string
cgofiles = append(cgofiles, pkg.CgoFiles...)
var imports []string
imports = append(imports, pkg.Package.Imports...)
imports = append(imports, pkg.Package.TestImports...)
name := pkg.Name
if name == "main" {
// rename the main package to its package name for testing.
name = filepath.Base(filepath.FromSlash(pkg.ImportPath))
}
// internal tests
testpkg, err := pkg.NewPackage(&importer.Package{
Name: name,
ImportPath: pkg.ImportPath,
Dir: pkg.Dir,
SrcRoot: pkg.SrcRoot,
GoFiles: gofiles,
CFiles: pkg.CFiles,
CgoFiles: cgofiles,
TestGoFiles: pkg.TestGoFiles, // passed directly to buildTestMain
XTestGoFiles: pkg.XTestGoFiles, // passed directly to buildTestMain
CgoCFLAGS: pkg.CgoCFLAGS,
CgoCPPFLAGS: pkg.CgoCPPFLAGS,
CgoCXXFLAGS: pkg.CgoCXXFLAGS,
CgoLDFLAGS: pkg.CgoLDFLAGS,
CgoPkgConfig: pkg.CgoPkgConfig,
Imports: imports,
})
if err != nil {
return nil, err
}
testpkg.TestScope = true
testpkg.Stale = true // TODO(dfc) NewPackage should get this right
// only build the internal test if there is Go source or
// internal test files.
var testobj *gb.Action
if len(testpkg.GoFiles)+len(testpkg.CgoFiles)+len(testpkg.TestGoFiles) > 0 {
// build internal testpkg dependencies
deps, err := gb.BuildDependencies(targets, testpkg)
if err != nil {
return nil, err
}
testobj, err = gb.Compile(testpkg, deps...)
if err != nil {
return nil, err
}
}
// external tests
if len(pkg.XTestGoFiles) > 0 {
xtestpkg, err := pkg.NewPackage(&importer.Package{
Name: name,
ImportPath: pkg.ImportPath + "_test",
Dir: pkg.Dir,
GoFiles: pkg.XTestGoFiles,
Imports: pkg.XTestImports,
})
if err != nil {
return nil, err
}
// build external test dependencies
deps, err := gb.BuildDependencies(targets, xtestpkg)
if err != nil {
return nil, err
}
xtestpkg.TestScope = true
xtestpkg.Stale = true
xtestpkg.ExtraIncludes = filepath.Join(pkg.Workdir(), filepath.FromSlash(pkg.ImportPath), "_test")
// if there is an internal test object, add it as a dependency.
if testobj != nil {
deps = append(deps, testobj)
}
testobj, err = gb.Compile(xtestpkg, deps...)
if err != nil {
return nil, err
}
}
testmainpkg, err := buildTestMain(testpkg)
if err != nil {
return nil, err
}
testmain, err := gb.Compile(testmainpkg, testobj)
if err != nil {
return nil, err
}
return &gb.Action{
Name: fmt.Sprintf("run: %s", testmainpkg.Binfile()),
Deps: testmain.Deps,
Run: func() error {
// When used with the concurrent executor, building deps and
// linking the test binary can cause a lot of disk space to be
// pinned as linking will tend to occur more frequenty than retiring
// tests.
//
// To solve this, we merge the testmain compile step (which includes
// linking) and the test run and cleanup steps so they are executed
// as one atomic operation.
var output bytes.Buffer
err := testmain.Run() // compile and link
if err == nil {
// nope mode means we stop at the compile and link phase.
if !pkg.Nope {
cmd := exec.Command(testmainpkg.Binfile(), flags...)
cmd.Dir = pkg.Dir // tests run in the original source directory
cmd.Stdout = &output
cmd.Stderr = &output
debug.Debugf("%s", cmd.Args)
err = cmd.Run() // run test
err = errors.Wrapf(err, "%s", cmd.Args) // wrap error if failed
}
// test binaries can be very large, so always unlink the
// binary after the test has run to free up temporary space
// technically this is done by ctx.Destroy(), but freeing
// the space earlier is important for projects with many
// packages
os.Remove(testmainpkg.Binfile())
}
if err != nil {
fmt.Fprintf(os.Stderr, "# %s\n", pkg.ImportPath)
} else {
fmt.Println(pkg.ImportPath)
}
if err != nil || pkg.Verbose {
io.Copy(os.Stdout, &output)
}
return err
},
}, nil
}
if err != nil {
return err
}
if dotfile != "" {
f, err := os.Create(dotfile)
if err != nil {
return err
}
defer f.Close()
printActions(f, test)
}
startSigHandlers()
return gb.ExecuteConcurrent(test, P, interrupted)
},
AddFlags: addTestFlags,
FlagParse: func(flags *flag.FlagSet, args []string) error {
var err error
debug.Debugf("%s", args)
args, tfs, err = TestFlagsExtraParse(args[2:])
debug.Debugf("%s %s", args, tfs)
if err != nil {
fmt.Fprintf(os.Stderr, "gb test: %s\n", err)
fmt.Fprintf(os.Stderr, `run "go help test" or "go help testflag" for more information`+"\n")
exit(2)
}
return flags.Parse(args)
},
}