// readManifest implements the ReadManifest logic and provides an
// optional flag that can be used to fetch the latest manifest updates
// from the manifest repository.
func readManifest(ctx *tool.Context, update bool) (Hosts, Projects, Tools, Hooks, error) {
ctx.TimerPush("read manifest")
defer ctx.TimerPop()
if update {
manifestPath, err := ToAbs(".manifest")
if err != nil {
return nil, nil, nil, nil, err
}
project := Project{
Path: manifestPath,
Protocol: "git",
Revision: "HEAD",
RemoteBranch: "master",
}
if err := resetProject(ctx, project); err != nil {
return nil, nil, nil, nil, err
}
}
path, err := ResolveManifestPath(ctx.Manifest())
if err != nil {
return nil, nil, nil, nil, err
}
hosts, projects, tools, hooks, stack := Hosts{}, Projects{}, Tools{}, Hooks{}, map[string]struct{}{}
if err := loadManifest(ctx, path, hosts, projects, tools, hooks, stack); err != nil {
return nil, nil, nil, nil, err
}
return hosts, projects, tools, hooks, nil
}
// UpdateUniverse updates all local projects and tools to match the
// remote counterparts identified by the given manifest. Optionally,
// the 'gc' flag can be used to indicate that local projects that no
// longer exist remotely should be removed.
func UpdateUniverse(ctx *tool.Context, gc bool) (e error) {
ctx.TimerPush("update universe")
defer ctx.TimerPop()
_, remoteProjects, remoteTools, remoteHooks, err := readManifest(ctx, true)
if err != nil {
return err
}
// 1. Update all local projects to match their remote counterparts.
if err := updateProjects(ctx, remoteProjects, gc); err != nil {
return err
}
// 2. Build all tools in a temporary directory.
tmpDir, err := ctx.Run().TempDir("", "tmp-jiri-tools-build")
if err != nil {
return fmt.Errorf("TempDir() failed: %v", err)
}
defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
if err := buildToolsFromMaster(ctx, remoteTools, tmpDir); err != nil {
return err
}
// 3. Install the tools into $JIRI_ROOT/devtools/bin.
if err := InstallTools(ctx, tmpDir); err != nil {
return err
}
// 4. Run all specified hooks
return runHooks(ctx, remoteHooks)
}
// InstallTools installs the tools from the given directory into
// $JIRI_ROOT/devtools/bin.
func InstallTools(ctx *tool.Context, dir string) error {
ctx.TimerPush("install tools")
defer ctx.TimerPop()
if ctx.DryRun() {
// In "dry run" mode, no binaries are built.
return nil
}
binDir, err := ToAbs(devtoolsBinDir)
if err != nil {
return err
}
fis, err := ioutil.ReadDir(dir)
if err != nil {
return fmt.Errorf("ReadDir(%v) failed: %v", dir, err)
}
failed := false
for _, fi := range fis {
installFn := func() error {
src := filepath.Join(dir, fi.Name())
dst := filepath.Join(binDir, fi.Name())
if err := ctx.Run().Rename(src, dst); err != nil {
return err
}
return nil
}
opts := runutil.Opts{Verbose: true}
if err := ctx.Run().FunctionWithOpts(opts, installFn, "install tool %q", fi.Name()); err != nil {
fmt.Fprintf(ctx.Stderr(), "%v\n", err)
failed = true
}
}
if failed {
return cmdline.ErrExitCode(2)
}
// Delete old "v23" tool, and the old jiri-xprofile command.
// TODO(nlacasse): Once everybody has had a chance to update, remove this
// code.
v23SubCmds := []string{
"jiri-xprofile",
"v23",
}
for _, subCmd := range v23SubCmds {
subCmdPath := filepath.Join(binDir, subCmd)
if err := ctx.Run().RemoveAll(subCmdPath); err != nil {
return err
}
}
return nil
}
// projectsExistLocally returns true iff all the given projects exist on the
// local filesystem.
// Note that this may return true even if there are projects on the local
// filesystem not included in the provided projects argument.
func projectsExistLocally(ctx *tool.Context, projects Projects) (bool, error) {
ctx.TimerPush("match manifest")
defer ctx.TimerPop()
for _, p := range projects {
isLocal, err := isLocalProject(ctx, p.Path)
if err != nil {
return false, err
}
if !isLocal {
return false, nil
}
}
return true, nil
}
// CreateSnapshot creates a manifest that encodes the current state of
// master branches of all projects and writes this snapshot out to the
// given file.
func CreateSnapshot(ctx *tool.Context, path string) error {
ctx.TimerPush("create snapshot")
defer ctx.TimerPop()
manifest := Manifest{}
// Add all local projects to manifest.
localProjects, err := LocalProjects(ctx, FullScan)
if err != nil {
return err
}
for _, project := range localProjects {
relPath, err := ToRel(project.Path)
if err != nil {
return err
}
project.Path = relPath
manifest.Projects = append(manifest.Projects, project)
}
// Add all hosts, tools, and hooks from the current manifest to the
// snapshot manifest.
hosts, _, tools, hooks, err := readManifest(ctx, true)
if err != nil {
return err
}
for _, tool := range tools {
manifest.Tools = append(manifest.Tools, tool)
}
for _, host := range hosts {
manifest.Hosts = append(manifest.Hosts, host)
}
for _, hook := range hooks {
manifest.Hooks = append(manifest.Hooks, hook)
}
perm := os.FileMode(0755)
if err := ctx.Run().MkdirAll(filepath.Dir(path), perm); err != nil {
return err
}
data, err := xml.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("MarshalIndent(%v) failed: %v", manifest, err)
}
perm = os.FileMode(0644)
if err := ctx.Run().WriteFile(path, data, perm); err != nil {
return err
}
return nil
}
// runHooks runs the specified hooks
func runHooks(ctx *tool.Context, hooks Hooks) error {
ctx.TimerPush("run hooks")
defer ctx.TimerPop()
for _, hook := range hooks {
command := hook.Path
args := []string{}
if hook.Interpreter != "" {
command = hook.Interpreter
args = append(args, hook.Path)
}
for _, arg := range hook.Args {
args = append(args, arg.Arg)
}
if err := ctx.Run().Command(command, args...); err != nil {
return fmt.Errorf("Hook %v failed: %v command: %v args: %v", hook.Name, err, command, args)
}
}
return nil
}
// LocalProjects returns projects on the local filesystem. If all projects in
// the manifest exist locally and scanMode is set to FastScan, then only the
// projects in the manifest that exist locally will be returned. Otherwise, a
// full scan of the filesystem will take place, and all found projects will be
// returned.
func LocalProjects(ctx *tool.Context, scanMode ScanMode) (Projects, error) {
ctx.TimerPush("local projects")
defer ctx.TimerPop()
projects := Projects{}
if scanMode == FastScan {
// Fast path: Full scan was not requested, and all projects in
// manifest exist on local filesystem. We just use the projects
// directly from the manifest.
manifestProjects, _, err := ReadManifest(ctx)
if err != nil {
return nil, err
}
projectsExist, err := projectsExistLocally(ctx, manifestProjects)
if err != nil {
return nil, err
}
if projectsExist {
return setProjectRevisions(ctx, manifestProjects)
}
}
// Slow path: Either full scan was not requested, or projects exist in
// manifest that were not found locally. Do a scan of all projects in
// JIRI_ROOT.
root, err := JiriRoot()
if err != nil {
return nil, err
}
// Initial call to findLocalProjects -- it will recursively search all the
// directories under JiriRoot.
ctx.TimerPush("scan fs")
err = findLocalProjects(ctx, root, projects)
ctx.TimerPop()
if err != nil {
return nil, err
}
return setProjectRevisions(ctx, projects)
}
// BuildTools builds the given tools and places the resulting binaries into the
// given directory.
func BuildTools(ctx *tool.Context, tools Tools, outputDir string) error {
ctx.TimerPush("build tools")
defer ctx.TimerPop()
if len(tools) == 0 {
// Nothing to do here...
return nil
}
projects, err := LocalProjects(ctx, FastScan)
if err != nil {
return err
}
toolPkgs := []string{}
workspaceSet := map[string]bool{}
for _, tool := range tools {
toolPkgs = append(toolPkgs, tool.Package)
toolProject, ok := projects[tool.Project]
if !ok {
return fmt.Errorf("project not found for tool %v", tool.Name)
}
// Identify the Go workspace the tool is in. To this end we use a
// heuristic that identifies the maximal suffix of the project path
// that corresponds to a prefix of the package name.
workspace := ""
for i := 0; i < len(toolProject.Path); i++ {
if toolProject.Path[i] == filepath.Separator {
if strings.HasPrefix("src/"+tool.Package, filepath.ToSlash(toolProject.Path[i+1:])) {
workspace = toolProject.Path[:i]
break
}
}
}
if workspace == "" {
return fmt.Errorf("could not identify go workspace for tool %v", tool.Name)
}
workspaceSet[workspace] = true
}
workspaces := []string{}
for workspace := range workspaceSet {
workspaces = append(workspaces, workspace)
}
if envGoPath := os.Getenv("GOPATH"); envGoPath != "" {
workspaces = append(workspaces, strings.Split(envGoPath, string(filepath.ListSeparator))...)
}
var stderr bytes.Buffer
opts := ctx.Run().Opts()
// We unset GOARCH and GOOS because jiri update should always build for the
// native architecture and OS. Also, as of go1.5, setting GOBIN is not
// compatible with GOARCH or GOOS.
opts.Env = map[string]string{
"GOARCH": "",
"GOOS": "",
"GOBIN": outputDir,
"GOPATH": strings.Join(workspaces, string(filepath.ListSeparator)),
}
opts.Stdout = ioutil.Discard
opts.Stderr = &stderr
args := append([]string{"install"}, toolPkgs...)
if err := ctx.Run().CommandWithOpts(opts, "go", args...); err != nil {
return fmt.Errorf("tool build failed\n%v", stderr.String())
}
return nil
}
// PollProjects returns the set of changelists that exist remotely but not
// locally. Changes are grouped by projects and contain author identification
// and a description of their content.
func PollProjects(ctx *tool.Context, projectSet map[string]struct{}) (_ Update, e error) {
ctx.TimerPush("poll projects")
defer ctx.TimerPop()
// Switch back to current working directory when we're done.
cwd, err := os.Getwd()
if err != nil {
return nil, err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
// Gather local & remote project data.
localProjects, err := LocalProjects(ctx, FastScan)
if err != nil {
return nil, err
}
_, remoteProjects, _, _, err := readManifest(ctx, false)
if err != nil {
return nil, err
}
// Compute difference between local and remote.
update := Update{}
ops, err := computeOperations(localProjects, remoteProjects, false)
if err != nil {
return nil, err
}
for _, op := range ops {
name := op.Project().Name
// If given a project set, limit our results to those projects in the set.
if len(projectSet) > 0 {
if _, ok := projectSet[name]; !ok {
continue
}
}
// We only inspect this project if an update operation is required.
cls := []CL{}
if updateOp, ok := op.(updateOperation); ok {
switch updateOp.project.Protocol {
case "git":
// Enter project directory - this assumes absolute paths.
if err := ctx.Run().Chdir(updateOp.destination); err != nil {
return nil, err
}
// Fetch the latest from remote.
if err := ctx.Git().FetchRefspec("origin", updateOp.project.RemoteBranch); err != nil {
return nil, err
}
// Collect commits visible from FETCH_HEAD that aren't visible from master.
commitsText, err := ctx.Git().Log("FETCH_HEAD", "master", "%an%n%ae%n%B")
if err != nil {
return nil, err
}
// Format those commits and add them to the results.
for _, commitText := range commitsText {
if got, want := len(commitText), 3; got < want {
return nil, fmt.Errorf("Unexpected length of %v: got %v, want at least %v", commitText, got, want)
}
cls = append(cls, CL{
Author: commitText[0],
Email: commitText[1],
Description: strings.Join(commitText[2:], "\n"),
})
}
default:
return nil, UnsupportedProtocolErr(updateOp.project.Protocol)
}
}
update[name] = cls
}
return update, nil
}
func updateProjects(ctx *tool.Context, remoteProjects Projects, gc bool) error {
ctx.TimerPush("update projects")
defer ctx.TimerPop()
scanMode := FastScan
if gc {
scanMode = FullScan
}
localProjects, err := LocalProjects(ctx, scanMode)
if err != nil {
return err
}
gitHost, gitHostErr := GitHost(ctx)
if gitHostErr == nil && googlesource.IsGoogleSourceHost(gitHost) {
// Attempt to get the repo statuses from remote so we can detect when a
// local project is already up-to-date.
if repoStatuses, err := googlesource.GetRepoStatuses(ctx, gitHost); err != nil {
// Log the error but don't fail.
fmt.Fprintf(ctx.Stderr(), "Error fetching repo statuses from remote: %v\n", err)
} else {
for name, rp := range remoteProjects {
status, ok := repoStatuses[rp.Name]
if !ok {
continue
}
masterRev, ok := status.Branches["master"]
if !ok || masterRev == "" {
continue
}
rp.Revision = masterRev
remoteProjects[name] = rp
}
}
}
ops, err := computeOperations(localProjects, remoteProjects, gc)
if err != nil {
return err
}
for _, op := range ops {
if err := op.Test(ctx); err != nil {
return err
}
}
failed := false
manifest := &Manifest{Label: ctx.Manifest()}
for _, op := range ops {
updateFn := func() error { return op.Run(ctx, manifest) }
// Always log the output of updateFn, irrespective of
// the value of the verbose flag.
opts := runutil.Opts{Verbose: true}
if err := ctx.Run().FunctionWithOpts(opts, updateFn, "%v", op); err != nil {
fmt.Fprintf(ctx.Stderr(), "%v\n", err)
failed = true
}
}
if failed {
return cmdline.ErrExitCode(2)
}
if err := writeCurrentManifest(ctx, manifest); err != nil {
return err
}
return nil
}