// MissingOSPackages returns the subset of the supplied packages that are
// missing from the underlying operating system and hence will need to
// be installed.
func MissingOSPackages(jirix *jiri.X, pkgs []string) ([]string, error) {
installedPkgs := map[string]bool{}
switch runtime.GOOS {
case "linux":
if IsFNLHost() {
fmt.Fprintf(jirix.Stdout(), "skipping %v on FNL host\n", pkgs)
break
}
var err error
installedPkgs, err = linuxList(jirix, pkgs)
if err != nil {
return nil, err
}
case "darwin":
var err error
installedPkgs, err = brewList(jirix)
if err != nil {
return nil, err
}
}
missing := []string{}
for _, pkg := range pkgs {
if !installedPkgs[pkg] {
missing = append(missing, pkg)
}
}
return missing, nil
}
func packagesImpl(jirix *jiri.X, cl *packagesFlagValues, args []string) error {
mgrs, _, err := availableProfileManagers(jirix, cl.dbPath, args)
if err != nil {
return err
}
cl.target.UseCommandLineEnv()
root := jiri.NewRelPath(cl.root).Join(profileInstaller)
s := jirix.NewSeq()
installPackages := cl.installPackages
// Never ask a subcommand to install packages.
cl.installPackages = false
for _, mgr := range mgrs {
cmds, err := mgr.packageCmds(jirix, cl, root)
if err != nil {
return err
}
for _, cmd := range cmds {
if installPackages {
if err := s.Verbose(true).Last(cmd[0], cmd[1:]...); err != nil {
return err
}
} else {
fmt.Fprintf(jirix.Stdout(), "%s\n", strings.TrimSpace(strings.Join(cmd, " ")))
}
}
}
return nil
}
func runUpdate(jirix *jiri.X, _ []string) error {
seq := jirix.NewSeq()
// Create the $JIRI_ROOT/.jiri_root directory if it doesn't already exist.
//
// TODO(toddw): Remove this logic after the transition to .jiri_root is done.
// The bootstrapping logic should create this directory, and jiri should fail
// if the directory doesn't exist.
if err := seq.MkdirAll(jirix.RootMetaDir(), 0755).Done(); err != nil {
return err
}
// Update all projects to their latest version.
// Attempt <attemptsFlag> times before failing.
updateFn := func() error { return project.UpdateUniverse(jirix, gcFlag) }
if err := retry.Function(jirix.Context, updateFn, retry.AttemptsOpt(attemptsFlag)); err != nil {
return err
}
if err := project.WriteUpdateHistorySnapshot(jirix, ""); err != nil {
return err
}
// Only attempt the bin dir transition after the update has succeeded, to
// avoid messy partial states.
return project.TransitionBinDir(jirix)
}
func linuxList(jirix *jiri.X, pkgs []string) (map[string]bool, error) {
aptitude, yum, pacman := UsingAptitude(jirix), UsingYum(jirix), UsingPacman(jirix)
cmd := ""
opt := ""
switch {
case aptitude:
cmd = "dpkg"
opt = "-L"
case yum:
cmd = "yum"
opt = "list"
case pacman:
cmd = "pacman"
opt = "-Q"
default:
return nil, fmt.Errorf("no usable package manager found, tested for aptitude, yum and pacman")
}
s := jirix.NewSeq()
installedPkgs := map[string]bool{}
for _, pkg := range pkgs {
if err := s.Capture(ioutil.Discard, ioutil.Discard).Last(cmd, opt, pkg); err == nil {
installedPkgs[pkg] = true
}
}
return installedPkgs, nil
}
func (ip *inproc) uninstall(jirix *jiri.X, cl *uninstallFlagValues, root jiri.RelPath) error {
profile := ip.db.LookupProfile(ip.installer, ip.name)
if profile == nil {
fmt.Fprintf(jirix.Stdout(), "%s is not installed\n", ip.qname)
return nil
}
mgr := profilesmanager.LookupManager(ip.qname)
var targets []*profiles.Target
if cl.allTargets {
targets = profile.Targets()
} else {
def, err := targetAtDefaultVersion(mgr, cl.target)
if err != nil {
return err
}
targets = []*profiles.Target{&def}
}
for _, target := range targets {
if err := mgr.Uninstall(jirix, ip.db, root, *target); err != nil {
logResult(jirix, "Uninstall", mgr, *target, err)
return err
}
logResult(jirix, "Uninstall", mgr, *target, nil)
}
return nil
}
func runRebuild(jirix *jiri.X, args []string) (e error) {
projects, tools, err := project.LoadManifest(jirix)
if err != nil {
return err
}
// Create a temporary directory in which tools will be built.
tmpDir, err := jirix.NewSeq().TempDir("", "tmp-jiri-rebuild")
if err != nil {
return fmt.Errorf("TempDir() failed: %v", err)
}
// Make sure we cleanup the temp directory.
defer collect.Error(func() error { return jirix.NewSeq().RemoveAll(tmpDir).Done() }, &e)
// Paranoid sanity checking.
if _, ok := tools[project.JiriName]; !ok {
return fmt.Errorf("tool %q not found", project.JiriName)
}
// Build and install tools.
if err := project.BuildTools(jirix, projects, tools, tmpDir); err != nil {
return err
}
return project.InstallTools(jirix, tmpDir)
}
func (ip *inproc) update(jirix *jiri.X, cl *updateFlagValues, root jiri.RelPath) error {
profile := ip.db.LookupProfile(ip.installer, ip.name)
if profile == nil {
// silently ignore uninstalled profile.
return nil
}
mgr := profilesmanager.LookupManager(ip.qname)
vi := mgr.VersionInfo()
for _, target := range profile.Targets() {
if vi.IsTargetOlderThanDefault(target.Version()) {
// Check if default target is already installed.
defTarget := *target
defTarget.SetVersion(vi.Default())
if profiles.FindTarget(profile.Targets(), &defTarget) != nil {
// Default target is already installed. Skip.
continue
}
if cl.verbose {
fmt.Fprintf(jirix.Stdout(), "Updating %s %s from %q to %s\n", ip.qname, target, target.Version(), vi)
}
err := mgr.Install(jirix, ip.db, root, defTarget)
logResult(jirix, "Update", mgr, defTarget, err)
if err != nil {
return err
}
} else {
if cl.verbose {
fmt.Fprintf(jirix.Stdout(), "%s %s at %q is up to date(%s)\n", ip.qname, target, target.Version(), vi)
}
}
}
return nil
}
func getCommitMessageFileName(jirix *jiri.X, branch string) (string, error) {
topLevel, err := gitutil.New(jirix.NewSeq()).TopLevel()
if err != nil {
return "", err
}
return filepath.Join(topLevel, jiri.ProjectMetaDir, branch, commitMessageFileName), nil
}
// checkDependents makes sure that all CLs in the sequence of
// dependent CLs leading to (but not including) the current branch
// have been exported to Gerrit.
func checkDependents(jirix *jiri.X) (e error) {
originalBranch, err := gitutil.New(jirix.NewSeq()).CurrentBranchName()
if err != nil {
return err
}
branches, err := getDependentCLs(jirix, originalBranch)
if err != nil {
return err
}
for i := 1; i < len(branches); i++ {
file, err := getCommitMessageFileName(jirix, branches[i])
if err != nil {
return err
}
if _, err := jirix.NewSeq().Stat(file); err != nil {
if !runutil.IsNotExist(err) {
return err
}
return fmt.Errorf(`Failed to export the branch %q to Gerrit because its ancestor %q has not been exported to Gerrit yet.
The following steps are needed before the operation can be retried:
$ git checkout %v
$ jiri cl mail
$ git checkout %v
# retry the original command
`, originalBranch, branches[i], branches[i], originalBranch)
}
}
return nil
}
func findProfileSubcommands(jirix *jiri.X) []string {
if !runSubcommands {
return nil
}
cmds, _ := lookpath.LookPrefix(jirix.Env(), "jiri-profile-", nil)
return cmds
}
func installImpl(jirix *jiri.X, cl *installFlagValues, args []string) error {
mgrs, db, err := availableProfileManagers(jirix, cl.dbPath, args)
if err != nil {
return err
}
cl.target.UseCommandLineEnv()
newMgrs := []profileManager{}
for _, mgr := range mgrs {
name := mgr.mgrName()
if !cl.force {
installer, profile := profiles.SplitProfileName(name)
if p := db.LookupProfileTarget(installer, profile, cl.target); p != nil {
fmt.Fprintf(jirix.Stdout(), "%v %v is already installed as %v\n", name, cl.target, p)
continue
}
}
newMgrs = append(newMgrs, mgr)
}
root := jiri.NewRelPath(cl.root).Join(profileInstaller)
for _, mgr := range newMgrs {
if err := mgr.install(jirix, cl, root); err != nil {
return err
}
}
return writeDB(jirix, db, profileInstaller, cl.dbPath)
}
// assertFilesNotCommitted asserts that the files exist and are *not*
// committed in the current branch.
func assertFilesNotCommitted(t *testing.T, jirix *jiri.X, files []string) {
assertFilesExist(t, jirix, files)
for _, file := range files {
if gitutil.New(jirix.NewSeq()).IsFileCommitted(file) {
t.Fatalf("expected file %v not to be committed but it is", file)
}
}
}
func newmapInput(jirix *jiri.X, state *project.ProjectState, key project.ProjectKey, index, total int) *mapInput {
return &mapInput{
ProjectState: state,
key: key,
jirix: jirix.Clone(tool.ContextOpts{}),
index: index,
total: total,
}
}
// Untar untars the file in srcFile and puts resulting files in directory dstDir.
func Untar(jirix *jiri.X, srcFile, dstDir string) error {
s := jirix.NewSeq()
if err := s.MkdirAll(dstDir, 0755).Done(); err != nil {
return err
}
return s.Output([]string{"untarring " + srcFile + " into " + dstDir}).
Pushd(dstDir).
Last("tar", "xvf", srcFile)
}
// assertFileContent asserts that the content of the given file
// matches the expected content.
func assertFileContent(t *testing.T, jirix *jiri.X, file, want string) {
got, err := jirix.NewSeq().ReadFile(file)
if err != nil {
t.Fatalf("%v\n", err)
}
if string(got) != want {
t.Fatalf("unexpected content of file %v: got %v, want %v", file, got, want)
}
}
// assertFilesPushedToRef asserts that the given files have been
// pushed to the given remote repository reference.
func assertFilesPushedToRef(t *testing.T, jirix *jiri.X, repoPath, gerritPath, pushedRef string, files []string) {
chdir(t, jirix, gerritPath)
assertCommitCount(t, jirix, pushedRef, "master", 1)
if err := gitutil.New(jirix.NewSeq()).CheckoutBranch(pushedRef); err != nil {
t.Fatalf("%v", err)
}
assertFilesCommitted(t, jirix, files)
chdir(t, jirix, repoPath)
}
// assertStashSize asserts that the stash size matches the expected
// size.
func assertStashSize(t *testing.T, jirix *jiri.X, want int) {
got, err := gitutil.New(jirix.NewSeq()).StashSize()
if err != nil {
t.Fatalf("%v", err)
}
if got != want {
t.Fatalf("unxpected stash size: got %v, want %v", got, want)
}
}
// assertCommitCount asserts that the commit count between two
// branches matches the expectedCount.
func assertCommitCount(t *testing.T, jirix *jiri.X, branch, baseBranch string, expectedCount int) {
got, err := gitutil.New(jirix.NewSeq()).CountCommits(branch, baseBranch)
if err != nil {
t.Fatalf("%v", err)
}
if want := 1; got != want {
t.Fatalf("unexpected number of commits: got %v, want %v", got, want)
}
}
// assertFilesDoNotExist asserts that the files do not exist.
func assertFilesDoNotExist(t *testing.T, jirix *jiri.X, files []string) {
s := jirix.NewSeq()
for _, file := range files {
if _, err := s.Stat(file); err != nil && !runutil.IsNotExist(err) {
t.Fatalf("%v", err)
} else if err == nil {
t.Fatalf("expected file %v to not exist but it did", file)
}
}
}
// commitFile commits a file with the specified content into a branch
func commitFile(t *testing.T, jirix *jiri.X, filename string, content string) {
s := jirix.NewSeq()
if err := s.WriteFile(filename, []byte(content), 0644).Done(); err != nil {
t.Fatalf("%v", err)
}
commitMessage := "Commit " + filename
if err := gitutil.New(jirix.NewSeq()).CommitFile(filename, commitMessage); err != nil {
t.Fatalf("%v", err)
}
}
// createRepoFromOrigin creates a Git repo tracking origin/master.
func createRepoFromOrigin(t *testing.T, jirix *jiri.X, subpath string, originPath string) string {
repoPath := createRepo(t, jirix, subpath)
chdir(t, jirix, repoPath)
if err := gitutil.New(jirix.NewSeq()).AddRemote("origin", originPath); err != nil {
t.Fatalf("%v", err)
}
if err := gitutil.New(jirix.NewSeq()).Pull("origin", "master"); err != nil {
t.Fatalf("%v", err)
}
return repoPath
}
// submit mocks a Gerrit review submit by pushing the Gerrit remote to origin.
// Actually origin pulls from Gerrit since origin isn't actually a bare git repo.
// Some of our tests actually rely on accessing .git in origin, so it must be non-bare.
func submit(t *testing.T, jirix *jiri.X, originPath string, gerritPath string, review *review) {
cwd, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd() failed: %v", err)
}
chdir(t, jirix, originPath)
expectedRef := gerrit.Reference(review.CLOpts)
if err := gitutil.New(jirix.NewSeq()).Pull(gerritPath, expectedRef); err != nil {
t.Fatalf("Pull gerrit to origin failed: %v", err)
}
chdir(t, jirix, cwd)
}
func brewList(jirix *jiri.X) (map[string]bool, error) {
var out bytes.Buffer
err := jirix.NewSeq().Capture(&out, &out).Last("brew", "list")
if err != nil || tool.VerboseFlag {
fmt.Fprintf(jirix.Stdout(), "%s", out.String())
}
scanner := bufio.NewScanner(&out)
pkgs := map[string]bool{}
for scanner.Scan() {
pkgs[scanner.Text()] = true
}
return pkgs, err
}
func checkReadme(t *testing.T, jirix *jiri.X, p project.Project, message string) {
if _, err := jirix.NewSeq().Stat(p.Path); err != nil {
t.Fatalf("%v", err)
}
readmeFile := filepath.Join(p.Path, "README")
data, err := ioutil.ReadFile(readmeFile)
if err != nil {
t.Fatalf("ReadFile(%v) failed: %v", readmeFile, err)
}
if got, want := data, []byte(message); bytes.Compare(got, want) != 0 {
t.Fatalf("unexpected content in project %v:\ngot\n%s\nwant\n%s\n", p.Name, got, want)
}
}
// Checks that /.jiri/ is ignored in a local project checkout
func checkMetadataIsIgnored(t *testing.T, jirix *jiri.X, p project.Project) {
if _, err := jirix.NewSeq().Stat(p.Path); err != nil {
t.Fatalf("%v", err)
}
gitInfoExcludeFile := filepath.Join(p.Path, ".git", "info", "exclude")
data, err := ioutil.ReadFile(gitInfoExcludeFile)
if err != nil {
t.Fatalf("ReadFile(%v) failed: %v", gitInfoExcludeFile, err)
}
excludeString := "/.jiri/"
if !strings.Contains(string(data), excludeString) {
t.Fatalf("Did not find \"%v\" in exclude file", excludeString)
}
}
func newCL(jirix *jiri.X, args []string) error {
git := gitutil.New(jirix.NewSeq())
topLevel, err := git.TopLevel()
if err != nil {
return err
}
originalBranch, err := git.CurrentBranchName()
if err != nil {
return err
}
// Create a new branch using the current branch.
newBranch := args[0]
if err := git.CreateAndCheckoutBranch(newBranch); err != nil {
return err
}
// Register a cleanup handler in case of subsequent errors.
cleanup := true
defer func() {
if cleanup {
git.CheckoutBranch(originalBranch, gitutil.ForceOpt(true))
git.DeleteBranch(newBranch, gitutil.ForceOpt(true))
}
}()
s := jirix.NewSeq()
// Record the dependent CLs for the new branch. The dependent CLs
// are recorded in a <dependencyPathFileName> file as a
// newline-separated list of branch names.
branches, err := getDependentCLs(jirix, originalBranch)
if err != nil {
return err
}
branches = append(branches, originalBranch)
newMetadataDir := filepath.Join(topLevel, jiri.ProjectMetaDir, newBranch)
if err := s.MkdirAll(newMetadataDir, os.FileMode(0755)).Done(); err != nil {
return err
}
file, err := getDependencyPathFileName(jirix, newBranch)
if err != nil {
return err
}
if err := s.WriteFile(file, []byte(strings.Join(branches, "\n")), os.FileMode(0644)).Done(); err != nil {
return err
}
cleanup = false
return nil
}
// AtomicAction performs an action 'atomically' by keeping track of successfully
// completed actions in the supplied completion log and re-running them if they
// are not successfully logged therein after deleting the entire contents of the
// dir parameter. Consequently it does not make sense to apply AtomicAction to
// the same directory in sequence.
func AtomicAction(jirix *jiri.X, installFn func() error, dir, message string) error {
atomicFn := func() error {
completionLogPath := filepath.Join(dir, ".complete")
s := jirix.NewSeq()
if dir != "" {
if exists, _ := s.IsDir(dir); exists {
// If the dir exists but the completionLogPath doesn't, then it
// means the previous action didn't finish.
// Remove the dir so we can perform the action again.
if exists, _ := s.IsFile(completionLogPath); !exists {
s.RemoveAll(dir).Done()
} else {
if jirix.Verbose() {
fmt.Fprintf(jirix.Stdout(), "AtomicAction: %s already completed in %s\n", message, dir)
}
return nil
}
}
}
if err := installFn(); err != nil {
if dir != "" {
s.RemoveAll(dir).Done()
}
return err
}
return s.WriteFile(completionLogPath, []byte("completed"), DefaultFilePerm).Done()
}
return jirix.NewSeq().Call(atomicFn, message).Done()
}
func checkReadme(t *testing.T, jirix *jiri.X, project, message string) {
s := jirix.NewSeq()
if _, err := s.Stat(project); err != nil {
t.Fatalf("%v", err)
}
readmeFile := filepath.Join(project, "README")
data, err := s.ReadFile(readmeFile)
if err != nil {
t.Fatalf("%v", err)
}
if got, want := data, []byte(message); bytes.Compare(got, want) != 0 {
t.Fatalf("unexpected content %v:\ngot\n%s\nwant\n%s\n", project, got, want)
}
}
// createTestRepos sets up three local repositories: origin, gerrit,
// and the main test repository which pulls from origin and can push
// to gerrit.
func createTestRepos(t *testing.T, jirix *jiri.X) (string, string, string) {
// Create origin.
originPath := createRepo(t, jirix, "origin")
chdir(t, jirix, originPath)
if err := gitutil.New(jirix.NewSeq()).CommitWithMessage("initial commit"); err != nil {
t.Fatalf("%v", err)
}
// Create test repo.
repoPath := createRepoFromOrigin(t, jirix, "test", originPath)
// Add Gerrit remote.
gerritPath := createRepoFromOrigin(t, jirix, "gerrit", originPath)
// Switch back to test repo.
chdir(t, jirix, repoPath)
return repoPath, originPath, gerritPath
}
func (sc *subcommand) packageCmds(jirix *jiri.X, cl *packagesFlagValues, root jiri.RelPath) ([][]string, error) {
cmd := []string{"profile-" + sc.installer, "os-packages"}
cmd = append(cmd, cl.args()...)
cmd = append(cmd, sc.qname)
var out bytes.Buffer
if err := jirix.NewSeq().Capture(&out, jirix.Stderr()).Last("jiri", cmd...); err != nil {
return nil, err
}
scanner := bufio.NewScanner(&out)
cmds := make([][]string, 0, 5)
for scanner.Scan() {
cmds = append(cmds, strings.Split(scanner.Text(), " "))
}
return cmds, nil
}