// readManifest implements the ReadManifest logic and provides an // optional flag that can be used to fetch the latest manifest updates // from the manifest repository. func readManifest(ctx *tool.Context, update bool) (Hosts, Projects, Tools, Hooks, error) { ctx.TimerPush("read manifest") defer ctx.TimerPop() if update { manifestPath, err := ToAbs(".manifest") if err != nil { return nil, nil, nil, nil, err } project := Project{ Path: manifestPath, Protocol: "git", Revision: "HEAD", RemoteBranch: "master", } if err := resetProject(ctx, project); err != nil { return nil, nil, nil, nil, err } } path, err := ResolveManifestPath(ctx.Manifest()) if err != nil { return nil, nil, nil, nil, err } hosts, projects, tools, hooks, stack := Hosts{}, Projects{}, Tools{}, Hooks{}, map[string]struct{}{} if err := loadManifest(ctx, path, hosts, projects, tools, hooks, stack); err != nil { return nil, nil, nil, nil, err } return hosts, projects, tools, hooks, nil }
// UpdateUniverse updates all local projects and tools to match the // remote counterparts identified by the given manifest. Optionally, // the 'gc' flag can be used to indicate that local projects that no // longer exist remotely should be removed. func UpdateUniverse(ctx *tool.Context, gc bool) (e error) { ctx.TimerPush("update universe") defer ctx.TimerPop() _, remoteProjects, remoteTools, remoteHooks, err := readManifest(ctx, true) if err != nil { return err } // 1. Update all local projects to match their remote counterparts. if err := updateProjects(ctx, remoteProjects, gc); err != nil { return err } // 2. Build all tools in a temporary directory. tmpDir, err := ctx.Run().TempDir("", "tmp-jiri-tools-build") if err != nil { return fmt.Errorf("TempDir() failed: %v", err) } defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e) if err := buildToolsFromMaster(ctx, remoteTools, tmpDir); err != nil { return err } // 3. Install the tools into $JIRI_ROOT/devtools/bin. if err := InstallTools(ctx, tmpDir); err != nil { return err } // 4. Run all specified hooks return runHooks(ctx, remoteHooks) }
// InstallTools installs the tools from the given directory into // $JIRI_ROOT/devtools/bin. func InstallTools(ctx *tool.Context, dir string) error { ctx.TimerPush("install tools") defer ctx.TimerPop() if ctx.DryRun() { // In "dry run" mode, no binaries are built. return nil } binDir, err := ToAbs(devtoolsBinDir) if err != nil { return err } fis, err := ioutil.ReadDir(dir) if err != nil { return fmt.Errorf("ReadDir(%v) failed: %v", dir, err) } failed := false for _, fi := range fis { installFn := func() error { src := filepath.Join(dir, fi.Name()) dst := filepath.Join(binDir, fi.Name()) if err := ctx.Run().Rename(src, dst); err != nil { return err } return nil } opts := runutil.Opts{Verbose: true} if err := ctx.Run().FunctionWithOpts(opts, installFn, "install tool %q", fi.Name()); err != nil { fmt.Fprintf(ctx.Stderr(), "%v\n", err) failed = true } } if failed { return cmdline.ErrExitCode(2) } // Delete old "v23" tool, and the old jiri-xprofile command. // TODO(nlacasse): Once everybody has had a chance to update, remove this // code. v23SubCmds := []string{ "jiri-xprofile", "v23", } for _, subCmd := range v23SubCmds { subCmdPath := filepath.Join(binDir, subCmd) if err := ctx.Run().RemoveAll(subCmdPath); err != nil { return err } } return nil }
// projectsExistLocally returns true iff all the given projects exist on the // local filesystem. // Note that this may return true even if there are projects on the local // filesystem not included in the provided projects argument. func projectsExistLocally(ctx *tool.Context, projects Projects) (bool, error) { ctx.TimerPush("match manifest") defer ctx.TimerPop() for _, p := range projects { isLocal, err := isLocalProject(ctx, p.Path) if err != nil { return false, err } if !isLocal { return false, nil } } return true, nil }
// CreateSnapshot creates a manifest that encodes the current state of // master branches of all projects and writes this snapshot out to the // given file. func CreateSnapshot(ctx *tool.Context, path string) error { ctx.TimerPush("create snapshot") defer ctx.TimerPop() manifest := Manifest{} // Add all local projects to manifest. localProjects, err := LocalProjects(ctx, FullScan) if err != nil { return err } for _, project := range localProjects { relPath, err := ToRel(project.Path) if err != nil { return err } project.Path = relPath manifest.Projects = append(manifest.Projects, project) } // Add all hosts, tools, and hooks from the current manifest to the // snapshot manifest. hosts, _, tools, hooks, err := readManifest(ctx, true) if err != nil { return err } for _, tool := range tools { manifest.Tools = append(manifest.Tools, tool) } for _, host := range hosts { manifest.Hosts = append(manifest.Hosts, host) } for _, hook := range hooks { manifest.Hooks = append(manifest.Hooks, hook) } perm := os.FileMode(0755) if err := ctx.Run().MkdirAll(filepath.Dir(path), perm); err != nil { return err } data, err := xml.MarshalIndent(manifest, "", " ") if err != nil { return fmt.Errorf("MarshalIndent(%v) failed: %v", manifest, err) } perm = os.FileMode(0644) if err := ctx.Run().WriteFile(path, data, perm); err != nil { return err } return nil }
// runHooks runs the specified hooks func runHooks(ctx *tool.Context, hooks Hooks) error { ctx.TimerPush("run hooks") defer ctx.TimerPop() for _, hook := range hooks { command := hook.Path args := []string{} if hook.Interpreter != "" { command = hook.Interpreter args = append(args, hook.Path) } for _, arg := range hook.Args { args = append(args, arg.Arg) } if err := ctx.Run().Command(command, args...); err != nil { return fmt.Errorf("Hook %v failed: %v command: %v args: %v", hook.Name, err, command, args) } } return nil }
// LocalProjects returns projects on the local filesystem. If all projects in // the manifest exist locally and scanMode is set to FastScan, then only the // projects in the manifest that exist locally will be returned. Otherwise, a // full scan of the filesystem will take place, and all found projects will be // returned. func LocalProjects(ctx *tool.Context, scanMode ScanMode) (Projects, error) { ctx.TimerPush("local projects") defer ctx.TimerPop() projects := Projects{} if scanMode == FastScan { // Fast path: Full scan was not requested, and all projects in // manifest exist on local filesystem. We just use the projects // directly from the manifest. manifestProjects, _, err := ReadManifest(ctx) if err != nil { return nil, err } projectsExist, err := projectsExistLocally(ctx, manifestProjects) if err != nil { return nil, err } if projectsExist { return setProjectRevisions(ctx, manifestProjects) } } // Slow path: Either full scan was not requested, or projects exist in // manifest that were not found locally. Do a scan of all projects in // JIRI_ROOT. root, err := JiriRoot() if err != nil { return nil, err } // Initial call to findLocalProjects -- it will recursively search all the // directories under JiriRoot. ctx.TimerPush("scan fs") err = findLocalProjects(ctx, root, projects) ctx.TimerPop() if err != nil { return nil, err } return setProjectRevisions(ctx, projects) }
// BuildTools builds the given tools and places the resulting binaries into the // given directory. func BuildTools(ctx *tool.Context, tools Tools, outputDir string) error { ctx.TimerPush("build tools") defer ctx.TimerPop() if len(tools) == 0 { // Nothing to do here... return nil } projects, err := LocalProjects(ctx, FastScan) if err != nil { return err } toolPkgs := []string{} workspaceSet := map[string]bool{} for _, tool := range tools { toolPkgs = append(toolPkgs, tool.Package) toolProject, ok := projects[tool.Project] if !ok { return fmt.Errorf("project not found for tool %v", tool.Name) } // Identify the Go workspace the tool is in. To this end we use a // heuristic that identifies the maximal suffix of the project path // that corresponds to a prefix of the package name. workspace := "" for i := 0; i < len(toolProject.Path); i++ { if toolProject.Path[i] == filepath.Separator { if strings.HasPrefix("src/"+tool.Package, filepath.ToSlash(toolProject.Path[i+1:])) { workspace = toolProject.Path[:i] break } } } if workspace == "" { return fmt.Errorf("could not identify go workspace for tool %v", tool.Name) } workspaceSet[workspace] = true } workspaces := []string{} for workspace := range workspaceSet { workspaces = append(workspaces, workspace) } if envGoPath := os.Getenv("GOPATH"); envGoPath != "" { workspaces = append(workspaces, strings.Split(envGoPath, string(filepath.ListSeparator))...) } var stderr bytes.Buffer opts := ctx.Run().Opts() // We unset GOARCH and GOOS because jiri update should always build for the // native architecture and OS. Also, as of go1.5, setting GOBIN is not // compatible with GOARCH or GOOS. opts.Env = map[string]string{ "GOARCH": "", "GOOS": "", "GOBIN": outputDir, "GOPATH": strings.Join(workspaces, string(filepath.ListSeparator)), } opts.Stdout = ioutil.Discard opts.Stderr = &stderr args := append([]string{"install"}, toolPkgs...) if err := ctx.Run().CommandWithOpts(opts, "go", args...); err != nil { return fmt.Errorf("tool build failed\n%v", stderr.String()) } return nil }
// PollProjects returns the set of changelists that exist remotely but not // locally. Changes are grouped by projects and contain author identification // and a description of their content. func PollProjects(ctx *tool.Context, projectSet map[string]struct{}) (_ Update, e error) { ctx.TimerPush("poll projects") defer ctx.TimerPop() // Switch back to current working directory when we're done. cwd, err := os.Getwd() if err != nil { return nil, err } defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e) // Gather local & remote project data. localProjects, err := LocalProjects(ctx, FastScan) if err != nil { return nil, err } _, remoteProjects, _, _, err := readManifest(ctx, false) if err != nil { return nil, err } // Compute difference between local and remote. update := Update{} ops, err := computeOperations(localProjects, remoteProjects, false) if err != nil { return nil, err } for _, op := range ops { name := op.Project().Name // If given a project set, limit our results to those projects in the set. if len(projectSet) > 0 { if _, ok := projectSet[name]; !ok { continue } } // We only inspect this project if an update operation is required. cls := []CL{} if updateOp, ok := op.(updateOperation); ok { switch updateOp.project.Protocol { case "git": // Enter project directory - this assumes absolute paths. if err := ctx.Run().Chdir(updateOp.destination); err != nil { return nil, err } // Fetch the latest from remote. if err := ctx.Git().FetchRefspec("origin", updateOp.project.RemoteBranch); err != nil { return nil, err } // Collect commits visible from FETCH_HEAD that aren't visible from master. commitsText, err := ctx.Git().Log("FETCH_HEAD", "master", "%an%n%ae%n%B") if err != nil { return nil, err } // Format those commits and add them to the results. for _, commitText := range commitsText { if got, want := len(commitText), 3; got < want { return nil, fmt.Errorf("Unexpected length of %v: got %v, want at least %v", commitText, got, want) } cls = append(cls, CL{ Author: commitText[0], Email: commitText[1], Description: strings.Join(commitText[2:], "\n"), }) } default: return nil, UnsupportedProtocolErr(updateOp.project.Protocol) } } update[name] = cls } return update, nil }
func updateProjects(ctx *tool.Context, remoteProjects Projects, gc bool) error { ctx.TimerPush("update projects") defer ctx.TimerPop() scanMode := FastScan if gc { scanMode = FullScan } localProjects, err := LocalProjects(ctx, scanMode) if err != nil { return err } gitHost, gitHostErr := GitHost(ctx) if gitHostErr == nil && googlesource.IsGoogleSourceHost(gitHost) { // Attempt to get the repo statuses from remote so we can detect when a // local project is already up-to-date. if repoStatuses, err := googlesource.GetRepoStatuses(ctx, gitHost); err != nil { // Log the error but don't fail. fmt.Fprintf(ctx.Stderr(), "Error fetching repo statuses from remote: %v\n", err) } else { for name, rp := range remoteProjects { status, ok := repoStatuses[rp.Name] if !ok { continue } masterRev, ok := status.Branches["master"] if !ok || masterRev == "" { continue } rp.Revision = masterRev remoteProjects[name] = rp } } } ops, err := computeOperations(localProjects, remoteProjects, gc) if err != nil { return err } for _, op := range ops { if err := op.Test(ctx); err != nil { return err } } failed := false manifest := &Manifest{Label: ctx.Manifest()} for _, op := range ops { updateFn := func() error { return op.Run(ctx, manifest) } // Always log the output of updateFn, irrespective of // the value of the verbose flag. opts := runutil.Opts{Verbose: true} if err := ctx.Run().FunctionWithOpts(opts, updateFn, "%v", op); err != nil { fmt.Fprintf(ctx.Stderr(), "%v\n", err) failed = true } } if failed { return cmdline.ErrExitCode(2) } if err := writeCurrentManifest(ctx, manifest); err != nil { return err } return nil }