// DisableRemoteManifestPush disables pushes to the remote manifest
// repository.
func (root FakeJiriRoot) DisableRemoteManifestPush(ctx *tool.Context) error {
dir := tool.RootDirOpt(filepath.Join(root.remote, manifestProject))
if err := ctx.Git(dir).CheckoutBranch("master"); err != nil {
return err
}
return nil
}
func createRemoteManifest(t *testing.T, ctx *tool.Context, dir string, remotes []string) {
manifestDir, perm := filepath.Join(dir, "v2"), os.FileMode(0755)
if err := ctx.Run().MkdirAll(manifestDir, perm); err != nil {
t.Fatalf("%v", err)
}
manifest := Manifest{}
for i, remote := range remotes {
project := Project{
Name: remote,
Path: localProjectName(i),
Protocol: "git",
Remote: remote,
}
manifest.Projects = append(manifest.Projects, project)
}
manifest.Hosts = []Host{
Host{
Name: "gerrit",
Location: "git://example.com/gerrit",
},
Host{
Name: "git",
Location: "git://example.com/git",
},
}
commitManifest(t, ctx, &manifest, dir)
}
func getManifest(ctx *tool.Context) string {
manifest := ctx.Manifest()
if manifest != "" {
return manifest
}
return defaultManifest
}
func createOncallFile(t *testing.T, ctx *tool.Context) {
content := `<?xml version="1.0" ?>
<rotation>
<shift>
<primary>spetrovic</primary>
<secondary>suharshs</secondary>
<startDate>Nov 5, 2014 12:00:00 PM</startDate>
</shift>
<shift>
<primary>suharshs</primary>
<secondary>jingjin</secondary>
<startDate>Nov 12, 2014 12:00:00 PM</startDate>
</shift>
<shift>
<primary>jsimsa</primary>
<secondary>toddw</secondary>
<startDate>Nov 19, 2014 12:00:00 PM</startDate>
</shift>
</rotation>`
oncallRotationsFile, err := OncallRotationPath(ctx)
if err != nil {
t.Fatalf("%v", err)
}
dir := filepath.Dir(oncallRotationsFile)
dirMode := os.FileMode(0700)
if err := ctx.Run().MkdirAll(dir, dirMode); err != nil {
t.Fatalf("MkdirAll(%q, %v) failed: %v", dir, dirMode, err)
}
fileMode := os.FileMode(0644)
if err := ioutil.WriteFile(oncallRotationsFile, []byte(content), fileMode); err != nil {
t.Fatalf("WriteFile(%q, %q, %v) failed: %v", oncallRotationsFile, content, fileMode, err)
}
}
func getDependencyPathFileName(ctx *tool.Context, branch string) (string, error) {
topLevel, err := ctx.Git().TopLevel()
if err != nil {
return "", err
}
return filepath.Join(topLevel, project.MetadataDirName(), branch, dependencyPathFileName), nil
}
// writeMetadata stores the given project metadata in the directory
// identified by the given path.
func writeMetadata(ctx *tool.Context, project Project, dir string) (e error) {
metadataDir := filepath.Join(dir, metadataDirName)
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().MkdirAll(metadataDir, os.FileMode(0755)); err != nil {
return err
}
if err := ctx.Run().Chdir(metadataDir); err != nil {
return err
}
// Replace absolute project paths with relative paths to make it
// possible to move the $JIRI_ROOT directory locally.
relPath, err := ToRel(project.Path)
if err != nil {
return err
}
project.Path = relPath
bytes, err := xml.Marshal(project)
if err != nil {
return fmt.Errorf("Marhsal() failed: %v", err)
}
metadataFile := filepath.Join(metadataDir, metadataFileName)
tmpMetadataFile := metadataFile + ".tmp"
if err := ctx.Run().WriteFile(tmpMetadataFile, bytes, os.FileMode(0644)); err != nil {
return err
}
if err := ctx.Run().Rename(tmpMetadataFile, metadataFile); err != nil {
return err
}
return nil
}
// gitCookies attempts to read and parse cookies from the .gitcookies file in
// the users home directory.
func gitCookies(ctx *tool.Context) []*http.Cookie {
cookies := []*http.Cookie{}
homeDir := os.Getenv("HOME")
if homeDir == "" {
return cookies
}
cookieFile := filepath.Join(homeDir, ".gitcookies")
bytes, err := ctx.Run().ReadFile(cookieFile)
if err != nil {
return cookies
}
lines := strings.Split(string(bytes), "\n")
for _, line := range lines {
if strings.TrimSpace(line) == "" {
continue
}
cookie, err := parseCookie(line)
if err != nil {
fmt.Fprintf(ctx.Stderr(), "error parsing cookie in .gitcookies: %v\n", err)
} else {
cookies = append(cookies, cookie)
}
}
return cookies
}
// setPathHelper is a utility function for setting path environment
// variables for different types of workspaces.
func setPathHelper(ctx *tool.Context, env *envvar.Vars, name, root string, workspaces []string, suffix string) error {
path := env.GetTokens(name, ":")
projects, _, err := project.ReadManifest(ctx)
if err != nil {
return err
}
for _, workspace := range workspaces {
absWorkspace := filepath.Join(root, workspace, suffix)
// Only append an entry to the path if the workspace is rooted
// under a jiri project that exists locally or vice versa.
for _, project := range projects {
// We check if <project.Path> is a prefix of <absWorkspace> to
// account for Go workspaces nested under a single jiri project,
// such as: $JIRI_ROOT/release/projects/chat/go.
//
// We check if <absWorkspace> is a prefix of <project.Path> to
// account for Go workspaces that span multiple jiri projects,
// such as: $JIRI_ROOT/release/go.
if strings.HasPrefix(absWorkspace, project.Path) || strings.HasPrefix(project.Path, absWorkspace) {
if _, err := ctx.Run().Stat(filepath.Join(absWorkspace)); err == nil {
path = append(path, absWorkspace)
break
}
}
}
}
env.SetTokens(name, path, ":")
return nil
}
func loadAliases(ctx *tool.Context) (*aliasMaps, error) {
aliasesFile := aliasesFlag
if aliasesFile == "" {
dataDir, err := project.DataDirPath(ctx, tool.Name)
if err != nil {
return nil, err
}
aliasesFile = filepath.Join(dataDir, aliasesFileName)
}
bytes, err := ctx.Run().ReadFile(aliasesFile)
if err != nil {
return nil, err
}
var data aliasesSchema
if err := xml.Unmarshal(bytes, &data); err != nil {
return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(bytes), err)
}
aliases := &aliasMaps{
emails: map[string]string{},
names: map[string]string{},
}
for _, email := range data.Emails {
for _, alias := range email.Aliases {
aliases.emails[alias] = email.Canonical
}
}
for _, name := range data.Names {
for _, alias := range name.Aliases {
aliases.names[alias] = name.Canonical
}
}
return aliases, nil
}
// checkDependents makes sure that all CLs in the sequence of
// dependent CLs leading to (but not including) the current branch
// have been exported to Gerrit.
func checkDependents(ctx *tool.Context) (e error) {
originalBranch, err := ctx.Git().CurrentBranchName()
if err != nil {
return err
}
branches, err := getDependentCLs(ctx, originalBranch)
if err != nil {
return err
}
for i := 1; i < len(branches); i++ {
file, err := getCommitMessageFileName(ctx, branches[i])
if err != nil {
return err
}
if _, err := ctx.Run().Stat(file); err != nil {
if !os.IsNotExist(err) {
return err
}
return fmt.Errorf(`Failed to export the branch %q to Gerrit because its ancestor %q has not been exported to Gerrit yet.
The following steps are needed before the operation can be retried:
$ git checkout %v
$ jiri cl mail
$ git checkout %v
# retry the original command
`, originalBranch, branches[i], branches[i], originalBranch)
}
}
return nil
}
// assertFilesNotCommitted asserts that the files exist and are *not*
// committed in the current branch.
func assertFilesNotCommitted(t *testing.T, ctx *tool.Context, files []string) {
assertFilesExist(t, ctx, files)
for _, file := range files {
if ctx.Git().IsFileCommitted(file) {
t.Fatalf("expected file %v not to be committed but it is", file)
}
}
}
func saveConfig(ctx *tool.Context, config *Config, path string) error {
var data configSchema
data.APICheckProjects = set.String.ToSlice(config.apiCheckProjects)
sort.Strings(data.APICheckProjects)
data.CopyrightCheckProjects = set.String.ToSlice(config.copyrightCheckProjects)
sort.Strings(data.CopyrightCheckProjects)
for _, workspace := range config.goWorkspaces {
data.GoWorkspaces = append(data.GoWorkspaces, workspace)
}
sort.Strings(data.GoWorkspaces)
for _, job := range config.jenkinsMatrixJobs {
data.JenkinsMatrixJobs = append(data.JenkinsMatrixJobs, job)
}
sort.Sort(data.JenkinsMatrixJobs)
for name, tests := range config.projectTests {
data.ProjectTests = append(data.ProjectTests, testGroupSchema{
Name: name,
Tests: tests,
})
}
sort.Sort(data.ProjectTests)
for name, dependencies := range config.testDependencies {
data.TestDependencies = append(data.TestDependencies, dependencyGroupSchema{
Name: name,
Dependencies: dependencies,
})
}
sort.Sort(data.TestDependencies)
for name, tests := range config.testGroups {
data.TestGroups = append(data.TestGroups, testGroupSchema{
Name: name,
Tests: tests,
})
}
sort.Sort(data.TestGroups)
for name, parts := range config.testParts {
data.TestParts = append(data.TestParts, partGroupSchema{
Name: name,
Parts: parts,
})
}
sort.Sort(data.TestParts)
for _, workspace := range config.vdlWorkspaces {
data.VDLWorkspaces = append(data.VDLWorkspaces, workspace)
}
sort.Strings(data.VDLWorkspaces)
bytes, err := xml.MarshalIndent(data, "", " ")
if err != nil {
return fmt.Errorf("MarshalIndent(%v) failed: %v", data, err)
}
if err := ctx.Run().MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil {
return err
}
if err := ctx.Run().WriteFile(path, bytes, os.FileMode(0644)); err != nil {
return err
}
return nil
}
// assertFilesDoNotExist asserts that the files do not exist.
func assertFilesDoNotExist(t *testing.T, ctx *tool.Context, files []string) {
for _, file := range files {
if _, err := ctx.Run().Stat(file); err != nil && !os.IsNotExist(err) {
t.Fatalf("%v", err)
} else if err == nil {
t.Fatalf("expected file %v to not exist but it did", file)
}
}
}
func (op deleteOperation) Test(ctx *tool.Context) error {
if _, err := ctx.Run().Stat(op.source); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("cannot delete %q as it does not exist", op.source)
}
return err
}
return nil
}
// assertCommitCount asserts that the commit count between two
// branches matches the expectedCount.
func assertCommitCount(t *testing.T, ctx *tool.Context, branch, baseBranch string, expectedCount int) {
got, err := ctx.Git().CountCommits(branch, baseBranch)
if err != nil {
t.Fatalf("%v", err)
}
if want := 1; got != want {
t.Fatalf("unexpected number of commits: got %v, want %v", got, want)
}
}
// commitFile commits a file with the specified content into a branch
func commitFile(t *testing.T, ctx *tool.Context, filename string, content string) {
if err := ctx.Run().WriteFile(filename, []byte(content), 0644); err != nil {
t.Fatalf("%v", err)
}
commitMessage := "Commit " + filename
if err := ctx.Git().CommitFile(filename, commitMessage); err != nil {
t.Fatalf("%v", err)
}
}
// assertFileContent asserts that the content of the given file
// matches the expected content.
func assertFileContent(t *testing.T, ctx *tool.Context, file, want string) {
got, err := ctx.Run().ReadFile(file)
if err != nil {
t.Fatalf("%v\n", err)
}
if string(got) != want {
t.Fatalf("unexpected content of file %v: got %v, want %v", file, got, want)
}
}
// assertStashSize asserts that the stash size matches the expected
// size.
func assertStashSize(t *testing.T, ctx *tool.Context, want int) {
got, err := ctx.Git().StashSize()
if err != nil {
t.Fatalf("%v", err)
}
if got != want {
t.Fatalf("unxpected stash size: got %v, want %v", got, want)
}
}
// assertFilesPushedToRef asserts that the given files have been
// pushed to the given remote repository reference.
func assertFilesPushedToRef(t *testing.T, ctx *tool.Context, repoPath, gerritPath, pushedRef string, files []string) {
chdir(t, ctx, gerritPath)
assertCommitCount(t, ctx, pushedRef, "master", 1)
if err := ctx.Git().CheckoutBranch(pushedRef); err != nil {
t.Fatalf("%v", err)
}
assertFilesCommitted(t, ctx, files)
chdir(t, ctx, repoPath)
}
// createRepoFromOrigin creates a Git repo tracking origin/master.
func createRepoFromOrigin(t *testing.T, ctx *tool.Context, workingDir string, subpath string, originPath string) string {
repoPath := createRepo(t, ctx, workingDir, subpath)
chdir(t, ctx, repoPath)
if err := ctx.Git().AddRemote("origin", originPath); err != nil {
t.Fatalf("%v", err)
}
if err := ctx.Git().Pull("origin", "master"); err != nil {
t.Fatalf("%v", err)
}
return repoPath
}
func (op createOperation) Test(ctx *tool.Context) error {
// Check the local file system.
if _, err := ctx.Run().Stat(op.destination); err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
return fmt.Errorf("cannot create %q as it already exists", op.destination)
}
return nil
}
func (root FakeJiriRoot) readManifest(ctx *tool.Context, path string) (*Manifest, error) {
bytes, err := ctx.Run().ReadFile(path)
if err != nil {
return nil, err
}
var manifest Manifest
if err := xml.Unmarshal(bytes, &manifest); err != nil {
return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(bytes), err)
}
return &manifest, nil
}
// submit mocks a Gerrit review submit by pushing the Gerrit remote to origin.
// Actually origin pulls from Gerrit since origin isn't actually a bare git repo.
// Some of our tests actually rely on accessing .git in origin, so it must be non-bare.
func submit(t *testing.T, ctx *tool.Context, originPath string, gerritPath string, review *review) {
cwd, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd() failed: %v", err)
}
chdir(t, ctx, originPath)
expectedRef := gerrit.Reference(review.CLOpts)
if err := ctx.Git().Pull(gerritPath, expectedRef); err != nil {
t.Fatalf("Pull gerrit to origin failed: %v", err)
}
chdir(t, ctx, cwd)
}
func checkReadme(t *testing.T, ctx *tool.Context, project, message string) {
if _, err := ctx.Run().Stat(project); err != nil {
t.Fatalf("%v", err)
}
readmeFile := filepath.Join(project, "README")
data, err := ctx.Run().ReadFile(readmeFile)
if err != nil {
t.Fatalf("%v", err)
}
if got, want := data, []byte(message); bytes.Compare(got, want) != 0 {
t.Fatalf("unexpected content %v:\ngot\n%s\nwant\n%s\n", project, got, want)
}
}
// WARNING: this function is in the proces of being removed and replaced
// by the profiles based configuration (see jiri/profiles). Use
// profiles.ConfigHelper instead of this for new code.
//
// JiriLegacyEnvironment returns the environment variables setting for the project.
// The util package captures the original state of the relevant environment
// variables when the tool is initialized and every invocation of this function
// updates this original state according to the jiri tool configuration.
//
// By default, the Go and VDL workspaces are added to the GOPATH and VDLPATH
// environment variables respectively. In addition, the JIRI_PROFILE
// environment variable can be used to activate an environment variable setting
// for various development profiles of the project (e.g. arm, android, java, or
// nacl). Unlike the default setting, the setting enabled by the JIRI_PROFILE
// environment variable can override existing environment.
func JiriLegacyEnvironment(ctx *tool.Context) (*envvar.Vars, error) {
env := envvar.VarsFromOS()
root, err := project.JiriRoot()
if err != nil {
return nil, err
}
config, err := LoadConfig(ctx)
if err != nil {
return nil, err
}
env.Set("CGO_ENABLED", "1")
if err := setGoPath(ctx, env, root, config); err != nil {
return nil, err
}
if err := setVdlPath(ctx, env, root, config); err != nil {
return nil, err
}
if profile := os.Getenv(jiriProfileEnv); profile != "" {
fmt.Fprintf(ctx.Stdout(), `NOTE: Enabling environment variable setting for %q.
This can override values of existing environment variables.
`, profile)
switch profile {
case "android":
// Cross-compilation for android on linux.
if err := setAndroidEnv(ctx, env, root); err != nil {
return nil, err
}
case "arm":
// Cross-compilation for arm on linux.
if err := setArmEnv(ctx, env, root); err != nil {
return nil, err
}
case "java":
// Building of a Go shared library for Java.
if err := setJavaEnv(ctx, env, root); err != nil {
return nil, err
}
case "nacl":
// Cross-compilation for nacl.
if err := setNaclEnv(ctx, env, root); err != nil {
return nil, err
}
default:
fmt.Fprintf(ctx.Stderr(), "Unknown environment profile %q", profile)
}
}
if err := setSyncbaseEnv(ctx, env, root); err != nil {
return nil, err
}
return env, nil
}
func (op deleteOperation) Run(ctx *tool.Context, _ *Manifest) error {
if op.gc {
// Never delete the <JiriProject>.
if op.project.Name == JiriProject {
lines := []string{
fmt.Sprintf("NOTE: project %v was not found in the project manifest", op.project.Name),
"however this project is required for correct operation of the jiri",
"development tools and will thus not be deleted",
}
opts := runutil.Opts{Verbose: true}
ctx.Run().OutputWithOpts(opts, lines)
return nil
}
// Never delete projects with non-master branches, uncommitted
// work, or untracked content.
git := ctx.Git(tool.RootDirOpt(op.project.Path))
branches, _, err := git.GetBranches()
if err != nil {
return err
}
uncommitted, err := git.HasUncommittedChanges()
if err != nil {
return err
}
untracked, err := git.HasUntrackedFiles()
if err != nil {
return err
}
if len(branches) != 1 || uncommitted || untracked {
lines := []string{
fmt.Sprintf("NOTE: project %v was not found in the project manifest", op.project.Name),
"however this project either contains non-master branches, uncommitted",
"work, or untracked files and will thus not be deleted",
}
opts := runutil.Opts{Verbose: true}
ctx.Run().OutputWithOpts(opts, lines)
return nil
}
return ctx.Run().RemoveAll(op.source)
}
lines := []string{
fmt.Sprintf("NOTE: project %v was not found in the project manifest", op.project.Name),
"it was not automatically removed to avoid deleting uncommitted work",
fmt.Sprintf(`if you no longer need it, invoke "rm -rf %v"`, op.source),
`or invoke "jiri update -gc" to remove all such local projects`,
}
opts := runutil.Opts{Verbose: true}
ctx.Run().OutputWithOpts(opts, lines)
return nil
}
// reportNonMaster checks if the given project is on master branch and
// if not, reports this fact along with information on how to update it.
func reportNonMaster(ctx *tool.Context, project Project) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().Chdir(project.Path); err != nil {
return err
}
switch project.Protocol {
case "git":
current, err := ctx.Git().CurrentBranchName()
if err != nil {
return err
}
if current != "master" {
line1 := fmt.Sprintf(`NOTE: "jiri update" only updates the "master" branch and the current branch is %q`, current)
line2 := fmt.Sprintf(`to update the %q branch once the master branch is updated, run "git merge master"`, current)
opts := runutil.Opts{Verbose: true}
ctx.Run().OutputWithOpts(opts, []string{line1, line2})
}
return nil
default:
return UnsupportedProtocolErr(project.Protocol)
}
}
// projectsExistLocally returns true iff all the given projects exist on the
// local filesystem.
// Note that this may return true even if there are projects on the local
// filesystem not included in the provided projects argument.
func projectsExistLocally(ctx *tool.Context, projects Projects) (bool, error) {
ctx.TimerPush("match manifest")
defer ctx.TimerPop()
for _, p := range projects {
isLocal, err := isLocalProject(ctx, p.Path)
if err != nil {
return false, err
}
if !isLocal {
return false, nil
}
}
return true, nil
}
// writeCurrentManifest writes the given manifest to a file that
// stores the result of the most recent "jiri update" invocation.
func writeCurrentManifest(ctx *tool.Context, manifest *Manifest) error {
currentManifestPath, err := ToAbs(currentManifestFileName)
if err != nil {
return err
}
bytes, err := xml.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("MarshalIndent(%v) failed: %v", manifest, err)
}
if err := ctx.Run().WriteFile(currentManifestPath, bytes, os.FileMode(0644)); err != nil {
return err
}
return nil
}
// UpdateUniverse updates all local projects and tools to match the
// remote counterparts identified by the given manifest. Optionally,
// the 'gc' flag can be used to indicate that local projects that no
// longer exist remotely should be removed.
func UpdateUniverse(ctx *tool.Context, gc bool) (e error) {
ctx.TimerPush("update universe")
defer ctx.TimerPop()
_, remoteProjects, remoteTools, remoteHooks, err := readManifest(ctx, true)
if err != nil {
return err
}
// 1. Update all local projects to match their remote counterparts.
if err := updateProjects(ctx, remoteProjects, gc); err != nil {
return err
}
// 2. Build all tools in a temporary directory.
tmpDir, err := ctx.Run().TempDir("", "tmp-jiri-tools-build")
if err != nil {
return fmt.Errorf("TempDir() failed: %v", err)
}
defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
if err := buildToolsFromMaster(ctx, remoteTools, tmpDir); err != nil {
return err
}
// 3. Install the tools into $JIRI_ROOT/devtools/bin.
if err := InstallTools(ctx, tmpDir); err != nil {
return err
}
// 4. Run all specified hooks
return runHooks(ctx, remoteHooks)
}
