// hostCredentials returns credentials for the given Gerrit host. The
// function uses best effort to scan common locations where the
// credentials could exist.
func hostCredentials(seq runutil.Sequence, hostUrl *url.URL) (_ *credentials, e error) {
// Look for the host credentials in the .netrc file.
netrcPath := filepath.Join(os.Getenv("HOME"), ".netrc")
file, err := seq.Open(netrcPath)
if err != nil {
if !runutil.IsNotExist(err) {
return nil, err
}
} else {
defer collect.Error(func() error { return file.Close() }, &e)
credsMap, err := parseNetrcFile(file)
if err != nil {
return nil, err
}
creds, ok := credsMap[hostUrl.Host]
if ok {
return creds, nil
}
}
// Look for the host credentials in the git cookie file.
args := []string{"config", "--get", "http.cookiefile"}
var stdout, stderr bytes.Buffer
if err := seq.Capture(&stdout, &stderr).Last("git", args...); err == nil {
cookieFilePath := strings.TrimSpace(stdout.String())
file, err := seq.Open(cookieFilePath)
if err != nil {
if !runutil.IsNotExist(err) {
return nil, err
}
} else {
defer collect.Error(func() error { return file.Close() }, &e)
credsMap, err := parseGitCookieFile(file)
if err != nil {
return nil, err
}
creds, ok := credsMap[hostUrl.Host]
if ok {
return creds, nil
}
// Account for site-wide credentials. Namely, the git cookie
// file can contain credentials of the form ".<name>", which
// should match any host "*.<name>".
for host, creds := range credsMap {
if strings.HasPrefix(host, ".") && strings.HasSuffix(hostUrl.Host, host) {
return creds, nil
}
}
}
}
return nil, fmt.Errorf("cannot find credentials for %q", hostUrl.String())
}
func runRebuild(jirix *jiri.X, args []string) (e error) {
projects, tools, err := project.LoadManifest(jirix)
if err != nil {
return err
}
// Create a temporary directory in which tools will be built.
tmpDir, err := jirix.NewSeq().TempDir("", "tmp-jiri-rebuild")
if err != nil {
return fmt.Errorf("TempDir() failed: %v", err)
}
// Make sure we cleanup the temp directory.
defer collect.Error(func() error { return jirix.NewSeq().RemoveAll(tmpDir).Done() }, &e)
// Paranoid sanity checking.
if _, ok := tools[project.JiriName]; !ok {
return fmt.Errorf("tool %q not found", project.JiriName)
}
// Build and install tools.
if err := project.BuildTools(jirix, projects, tools, tmpDir); err != nil {
return err
}
return project.InstallTools(jirix, tmpDir)
}
// commitAndPushChanges commits changes identified by the given manifest file
// and label to the containing repository and pushes these changes to the
// remote repository.
func commitAndPushChanges(jirix *jiri.X, snapshotDir, snapshotFile, label string) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return jirix.NewSeq().Chdir(cwd).Done() }, &e)
if err := jirix.NewSeq().Chdir(snapshotDir).Done(); err != nil {
return err
}
relativeSnapshotPath := strings.TrimPrefix(snapshotFile, snapshotDir+string(os.PathSeparator))
git := gitutil.New(jirix.NewSeq())
// Pull from master so we are up-to-date.
if err := git.Pull("origin", "master"); err != nil {
return err
}
if err := git.Add(relativeSnapshotPath); err != nil {
return err
}
if err := git.Add(label); err != nil {
return err
}
name := strings.TrimPrefix(snapshotFile, snapshotDir)
if err := git.CommitNoVerify(fmt.Sprintf("adding snapshot %q for label %q", name, label)); err != nil {
return err
}
if err := git.Push("origin", "master", gitutil.VerifyOpt(false)); err != nil {
return err
}
return nil
}
// writeMetadata stores the given project metadata in the directory
// identified by the given path.
func writeMetadata(ctx *tool.Context, project Project, dir string) (e error) {
metadataDir := filepath.Join(dir, metadataDirName)
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().MkdirAll(metadataDir, os.FileMode(0755)); err != nil {
return err
}
if err := ctx.Run().Chdir(metadataDir); err != nil {
return err
}
// Replace absolute project paths with relative paths to make it
// possible to move the $JIRI_ROOT directory locally.
relPath, err := ToRel(project.Path)
if err != nil {
return err
}
project.Path = relPath
bytes, err := xml.Marshal(project)
if err != nil {
return fmt.Errorf("Marhsal() failed: %v", err)
}
metadataFile := filepath.Join(metadataDir, metadataFileName)
tmpMetadataFile := metadataFile + ".tmp"
if err := ctx.Run().WriteFile(tmpMetadataFile, bytes, os.FileMode(0644)); err != nil {
return err
}
if err := ctx.Run().Rename(tmpMetadataFile, metadataFile); err != nil {
return err
}
return nil
}
// invoke invokes the Jenkins API using the given suffix, values and
// HTTP method.
func (j *Jenkins) invoke(method, suffix string, values url.Values) (_ []byte, err error) {
// Return mock result in test mode.
if j.testMode {
return j.invokeMockResults[suffix], nil
}
apiURL, err := url.Parse(j.host)
if err != nil {
return nil, fmt.Errorf("Parse(%q) failed: %v", j.host, err)
}
apiURL.Path = fmt.Sprintf("%s/%s", apiURL.Path, suffix)
apiURL.RawQuery = values.Encode()
var body io.Reader
url, body := apiURL.String(), nil
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, fmt.Errorf("NewRequest(%q, %q, %v) failed: %v", method, url, body, err)
}
req.Header.Add("Accept", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("Do(%v) failed: %v", req, err)
}
defer collect.Error(func() error { return res.Body.Close() }, &err)
bytes, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
// queue/cancelItem API returns 404 even successful.
// See: https://issues.jenkins-ci.org/browse/JENKINS-21311.
if suffix != "queue/cancelItem" && res.StatusCode >= http.StatusBadRequest {
return nil, fmt.Errorf("HTTP request %q returned %d:\n%s", url, res.StatusCode, string(bytes))
}
return bytes, nil
}
// setProjectRevisions sets the current project revision from the master for
// each project as found on the filesystem
func setProjectRevisions(ctx *tool.Context, projects Projects) (_ Projects, e error) {
cwd, err := os.Getwd()
if err != nil {
return nil, err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
for name, project := range projects {
switch project.Protocol {
case "git":
if err := ctx.Run().Chdir(project.Path); err != nil {
return nil, err
}
revision, err := ctx.Git().CurrentRevisionOfBranch("master")
if err != nil {
return nil, err
}
project.Revision = revision
default:
return nil, UnsupportedProtocolErr(project.Protocol)
}
projects[name] = project
}
return projects, nil
}
// SetTopic sets the topic of the given Gerrit reference.
func (g *Gerrit) SetTopic(cl string, opts CLOpts) (e error) {
cred, err := hostCredentials(g.r, g.host)
if err != nil {
return err
}
topic := Topic{opts.Topic}
data, err := json.Marshal(topic)
if err != nil {
return fmt.Errorf("Marshal(%#v) failed: %v", topic, err)
}
url := fmt.Sprintf("%s/a/changes/%s/topic", g.host, cl)
method, body := "PUT", bytes.NewReader(data)
req, err := http.NewRequest(method, url, body)
if err != nil {
return fmt.Errorf("NewRequest(%q, %q, %v) failed: %v", method, url, body, err)
}
req.Header.Add("Content-Type", "application/json;charset=UTF-8")
req.SetBasicAuth(cred.username, cred.password)
res, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("Do(%v) failed: %v", req, err)
}
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusNoContent {
return fmt.Errorf("SetTopic:Do(%v) failed: %v", req, res.StatusCode)
}
defer collect.Error(func() error { return res.Body.Close() }, &e)
return nil
}
// UpdateUniverse updates all local projects and tools to match the
// remote counterparts identified by the given manifest. Optionally,
// the 'gc' flag can be used to indicate that local projects that no
// longer exist remotely should be removed.
func UpdateUniverse(ctx *tool.Context, gc bool) (e error) {
ctx.TimerPush("update universe")
defer ctx.TimerPop()
_, remoteProjects, remoteTools, remoteHooks, err := readManifest(ctx, true)
if err != nil {
return err
}
// 1. Update all local projects to match their remote counterparts.
if err := updateProjects(ctx, remoteProjects, gc); err != nil {
return err
}
// 2. Build all tools in a temporary directory.
tmpDir, err := ctx.Run().TempDir("", "tmp-jiri-tools-build")
if err != nil {
return fmt.Errorf("TempDir() failed: %v", err)
}
defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
if err := buildToolsFromMaster(ctx, remoteTools, tmpDir); err != nil {
return err
}
// 3. Install the tools into $JIRI_ROOT/devtools/bin.
if err := InstallTools(ctx, tmpDir); err != nil {
return err
}
// 4. Run all specified hooks
return runHooks(ctx, remoteHooks)
}
// Query returns a list of QueryResult entries matched by the given
// Gerrit query string from the given Gerrit instance. The result is
// sorted by the last update time, most recently updated to oldest
// updated.
//
// See the following links for more details about Gerrit search syntax:
// - https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
// - https://gerrit-review.googlesource.com/Documentation/user-search.html
func (g *Gerrit) Query(query string) (_ []Change, e error) {
cred, err := hostCredentials(g.r, g.host)
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/a/changes/?o=CURRENT_REVISION&o=CURRENT_COMMIT&o=LABELS&o=DETAILED_ACCOUNTS&q=%s", g.host, url.QueryEscape(query))
var body io.Reader
method, body := "GET", nil
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, fmt.Errorf("NewRequest(%q, %q, %v) failed: %v", method, url, body, err)
}
req.Header.Add("Accept", "application/json")
req.SetBasicAuth(cred.username, cred.password)
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("Do(%v) failed: %v", req, err)
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Query:Do(%v) failed: %v", req, res.StatusCode)
}
defer collect.Error(func() error { return res.Body.Close() }, &e)
return parseQueryResults(res.Body)
}
// revisionChanges commits changes identified by the given manifest
// file and label to the manifest repository and (if applicable)
// pushes these changes to the remote repository.
func revisionChanges(ctx *tool.Context, snapshotDir, snapshotFile, label string) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().Chdir(snapshotDir); err != nil {
return err
}
relativeSnapshotPath := strings.TrimPrefix(snapshotFile, snapshotDir+string(os.PathSeparator))
if err := ctx.Git().Add(relativeSnapshotPath); err != nil {
return err
}
if err := ctx.Git().Add(label); err != nil {
return err
}
name := strings.TrimPrefix(snapshotFile, snapshotDir)
if err := ctx.Git().CommitWithMessage(fmt.Sprintf("adding snapshot %q for label %q", name, label)); err != nil {
return err
}
if remoteFlag {
if err := ctx.Git().Push("origin", "master", gitutil.VerifyOpt(false)); err != nil {
return err
}
}
return nil
}
// reportNonMaster checks if the given project is on master branch and
// if not, reports this fact along with information on how to update it.
func reportNonMaster(ctx *tool.Context, project Project) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().Chdir(project.Path); err != nil {
return err
}
switch project.Protocol {
case "git":
current, err := ctx.Git().CurrentBranchName()
if err != nil {
return err
}
if current != "master" {
line1 := fmt.Sprintf(`NOTE: "jiri update" only updates the "master" branch and the current branch is %q`, current)
line2 := fmt.Sprintf(`to update the %q branch once the master branch is updated, run "git merge master"`, current)
opts := runutil.Opts{Verbose: true}
ctx.Run().OutputWithOpts(opts, []string{line1, line2})
}
return nil
default:
return UnsupportedProtocolErr(project.Protocol)
}
}
func runRebuild(env *cmdline.Env, args []string) (e error) {
ctx := tool.NewContextFromEnv(env)
_, tools, err := project.ReadManifest(ctx)
if err != nil {
return err
}
// Create a temporary directory in which tools will be built.
tmpDir, err := ctx.Run().TempDir("", "tmp-jiri-rebuild")
if err != nil {
return fmt.Errorf("TempDir() failed: %v", err)
}
// Make sure we cleanup the temp directory.
defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
// Paranoid sanity checking.
if _, ok := tools[project.JiriName]; !ok {
return fmt.Errorf("tool %q not found", project.JiriName)
}
// Build and install tools.
if err := project.BuildTools(ctx, tools, tmpDir); err != nil {
return err
}
return project.InstallTools(ctx, tmpDir)
}
// ApplyToLocalMaster applies an operation expressed as the given function to
// the local master branch of the given projects.
func ApplyToLocalMaster(ctx *tool.Context, projects Projects, fn func() error) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
// Loop through all projects, checking out master and stashing any unstaged
// changes.
for _, project := range projects {
p := project
if err := ctx.Run().Chdir(p.Path); err != nil {
return err
}
switch p.Protocol {
case "git":
branch, err := ctx.Git().CurrentBranchName()
if err != nil {
return err
}
stashed, err := ctx.Git().Stash()
if err != nil {
return err
}
if err := ctx.Git().CheckoutBranch("master"); err != nil {
return err
}
// After running the function, return to this project's directory,
// checkout the original branch, and stash pop if necessary.
defer collect.Error(func() error {
if err := ctx.Run().Chdir(p.Path); err != nil {
return err
}
if err := ctx.Git().CheckoutBranch(branch); err != nil {
return err
}
if stashed {
return ctx.Git().StashPop()
}
return nil
}, &e)
default:
return UnsupportedProtocolErr(p.Protocol)
}
}
return fn()
}
func listCommitters(ctx *tool.Context) (_ []string, e error) {
branch, err := ctx.Git().CurrentBranchName()
if err != nil {
return nil, err
}
stashed, err := ctx.Git().Stash()
if err != nil {
return nil, err
}
if stashed {
defer collect.Error(func() error { return ctx.Git().StashPop() }, &e)
}
if err := ctx.Git().CheckoutBranch("master"); err != nil {
return nil, err
}
defer collect.Error(func() error { return ctx.Git().CheckoutBranch(branch) }, &e)
return ctx.Git().Committers()
}
// Submit submits the given changelist through Gerrit.
func (g *Gerrit) Submit(changeID string) (e error) {
cred, err := hostCredentials(g.r, g.host)
if err != nil {
return err
}
// Encode data needed for Submit.
data := struct {
WaitForMerge bool `json:"wait_for_merge"`
}{
WaitForMerge: true,
}
encodedBytes, err := json.Marshal(data)
if err != nil {
return fmt.Errorf("Marshal(%#v) failed: %v", data, err)
}
// Call Submit API.
// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#submit-change
url := fmt.Sprintf("%s/a/changes/%s/submit", g.host, changeID)
var body io.Reader
method, body := "POST", bytes.NewReader(encodedBytes)
req, err := http.NewRequest(method, url, body)
if err != nil {
return fmt.Errorf("NewRequest(%q, %q, %v) failed: %v", method, url, body, err)
}
req.Header.Add("Content-Type", "application/json;charset=UTF-8")
req.SetBasicAuth(cred.username, cred.password)
res, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("Do(%v) failed: %v", req, err)
}
if res.StatusCode != http.StatusOK {
return fmt.Errorf("Submit:Do(%v) failed: %v", req, res.StatusCode)
}
defer collect.Error(func() error { return res.Body.Close() }, &e)
// Check response.
bytes, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
resContent := string(bytes)
// For a "TBR" CL, the response code is not 200 but the submit will still succeed.
// In those cases, the "error" message will be "change is new".
// We don't treat this case as error.
if res.StatusCode != http.StatusOK && strings.TrimSpace(resContent) != "change is new" {
return fmt.Errorf("Failed to submit CL %q:\n%s", changeID, resContent)
}
return nil
}
func cleanupCL(jirix *jiri.X, branches []string) (e error) {
git := gitutil.New(jirix.NewSeq())
originalBranch, err := git.CurrentBranchName()
if err != nil {
return err
}
stashed, err := git.Stash()
if err != nil {
return err
}
if stashed {
defer collect.Error(func() error { return git.StashPop() }, &e)
}
if err := git.CheckoutBranch(remoteBranchFlag); err != nil {
return err
}
checkoutOriginalBranch := true
defer collect.Error(func() error {
if checkoutOriginalBranch {
return git.CheckoutBranch(originalBranch)
}
return nil
}, &e)
if err := git.FetchRefspec("origin", remoteBranchFlag); err != nil {
return err
}
s := jirix.NewSeq()
for _, branch := range branches {
cleanupFn := func() error { return cleanupBranch(jirix, branch) }
if err := s.Call(cleanupFn, "Cleaning up branch: %s", branch).Done(); err != nil {
return err
}
if branch == originalBranch {
checkoutOriginalBranch = false
}
}
return nil
}
func cleanupCL(ctx *tool.Context, branches []string) (e error) {
originalBranch, err := ctx.Git().CurrentBranchName()
if err != nil {
return err
}
stashed, err := ctx.Git().Stash()
if err != nil {
return err
}
if stashed {
defer collect.Error(func() error { return ctx.Git().StashPop() }, &e)
}
if err := ctx.Git().CheckoutBranch(remoteBranchFlag); err != nil {
return err
}
checkoutOriginalBranch := true
defer collect.Error(func() error {
if checkoutOriginalBranch {
return ctx.Git().CheckoutBranch(originalBranch)
}
return nil
}, &e)
if err := ctx.Git().FetchRefspec("origin", remoteBranchFlag); err != nil {
return err
}
for _, branch := range branches {
cleanupFn := func() error { return cleanupBranch(ctx, branch) }
if err := ctx.Run().Function(cleanupFn, "Cleaning up branch %q", branch); err != nil {
return err
}
if branch == originalBranch {
checkoutOriginalBranch = false
}
}
return nil
}
// CleanupProjects restores the given jiri projects back to their master
// branches and gets rid of all the local changes. If "cleanupBranches" is
// true, it will also delete all the non-master branches.
func CleanupProjects(ctx *tool.Context, projects Projects, cleanupBranches bool) (e error) {
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("Getwd() failed: %v", err)
}
defer collect.Error(func() error { return ctx.Run().Chdir(wd) }, &e)
for _, project := range projects {
localProjectDir := project.Path
if err := ctx.Run().Chdir(localProjectDir); err != nil {
return err
}
if err := resetLocalProject(ctx, cleanupBranches, project.RemoteBranch); err != nil {
return err
}
}
return nil
}
// PostReview posts a review to the given Gerrit reference.
func (g *Gerrit) PostReview(ref string, message string, labels map[string]string) (e error) {
cred, err := hostCredentials(g.r, g.host)
if err != nil {
return err
}
review := Review{
Message: message,
Labels: labels,
}
// Encode "review" as JSON.
encodedBytes, err := json.Marshal(review)
if err != nil {
return fmt.Errorf("Marshal(%#v) failed: %v", review, err)
}
// Construct API URL.
// ref is in the form of "refs/changes/<last two digits of change number>/<change number>/<patch set number>".
parts := strings.Split(ref, "/")
if expected, got := 5, len(parts); expected != got {
return fmt.Errorf("unexpected number of %q parts: expected %v, got %v", ref, expected, got)
}
cl, revision := parts[3], parts[4]
url := fmt.Sprintf("%s/a/changes/%s/revisions/%s/review", g.host, cl, revision)
// Post the review.
method, body := "POST", bytes.NewReader(encodedBytes)
req, err := http.NewRequest(method, url, body)
if err != nil {
return fmt.Errorf("NewRequest(%q, %q, %v) failed: %v", method, url, body, err)
}
req.Header.Add("Content-Type", "application/json;charset=UTF-8")
req.SetBasicAuth(cred.username, cred.password)
res, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("Do(%v) failed: %v", req, err)
}
if res.StatusCode != http.StatusOK {
return fmt.Errorf("PostReview:Do(%v) failed: %v", req, res.StatusCode)
}
defer collect.Error(func() error { return res.Body.Close() }, &e)
return nil
}
// checkSnapshotDir makes sure that he local snapshot directory exists
// and is initialized properly.
func checkSnapshotDir(ctx *tool.Context) (e error) {
snapshotDir, err := getSnapshotDir()
if err != nil {
return err
}
if _, err := ctx.Run().Stat(snapshotDir); err != nil {
if !os.IsNotExist(err) {
return err
}
if remoteFlag {
if err := ctx.Run().MkdirAll(snapshotDir, 0755); err != nil {
return err
}
return nil
}
createFn := func() (err error) {
if err := ctx.Run().MkdirAll(snapshotDir, 0755); err != nil {
return err
}
if err := ctx.Git().Init(snapshotDir); err != nil {
return err
}
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().Chdir(snapshotDir); err != nil {
return err
}
if err := ctx.Git().Commit(); err != nil {
return err
}
return nil
}
if err := createFn(); err != nil {
ctx.Run().RemoveAll(snapshotDir)
return err
}
}
return nil
}
// GitCloneRepo clones a repo at a specific revision in outDir.
func GitCloneRepo(ctx *tool.Context, remote, revision, outDir string, outDirPerm os.FileMode) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().MkdirAll(outDir, outDirPerm); err != nil {
return err
}
if err := ctx.Git().Clone(remote, outDir); err != nil {
return err
}
if err := ctx.Run().Chdir(outDir); err != nil {
return err
}
if err := ctx.Git().Reset(revision); err != nil {
return err
}
return nil
}
// WriteLog writes the given list of CLs to a log file, as a json-encoded
// map of ref strings => CLs.
func WriteLog(logFilePath string, cls CLList) (e error) {
// Index CLs with their refs.
results := CLRefMap{}
for _, cl := range cls {
results[cl.Reference()] = cl
}
fd, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
return fmt.Errorf("OpenFile(%q) failed: %v", logFilePath, err)
}
defer collect.Error(func() error { return fd.Close() }, &e)
bytes, err := json.MarshalIndent(results, "", " ")
if err != nil {
return fmt.Errorf("MarshalIndent(%v) failed: %v", results, err)
}
if err := ioutil.WriteFile(logFilePath, bytes, os.FileMode(0644)); err != nil {
return fmt.Errorf("WriteFile(%q) failed: %v", logFilePath, err)
}
return nil
}
// PollProjects returns the set of changelists that exist remotely but not
// locally. Changes are grouped by projects and contain author identification
// and a description of their content.
func PollProjects(ctx *tool.Context, projectSet map[string]struct{}) (_ Update, e error) {
ctx.TimerPush("poll projects")
defer ctx.TimerPop()
// Switch back to current working directory when we're done.
cwd, err := os.Getwd()
if err != nil {
return nil, err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
// Gather local & remote project data.
localProjects, err := LocalProjects(ctx, FastScan)
if err != nil {
return nil, err
}
_, remoteProjects, _, _, err := readManifest(ctx, false)
if err != nil {
return nil, err
}
// Compute difference between local and remote.
update := Update{}
ops, err := computeOperations(localProjects, remoteProjects, false)
if err != nil {
return nil, err
}
for _, op := range ops {
name := op.Project().Name
// If given a project set, limit our results to those projects in the set.
if len(projectSet) > 0 {
if _, ok := projectSet[name]; !ok {
continue
}
}
// We only inspect this project if an update operation is required.
cls := []CL{}
if updateOp, ok := op.(updateOperation); ok {
switch updateOp.project.Protocol {
case "git":
// Enter project directory - this assumes absolute paths.
if err := ctx.Run().Chdir(updateOp.destination); err != nil {
return nil, err
}
// Fetch the latest from remote.
if err := ctx.Git().FetchRefspec("origin", updateOp.project.RemoteBranch); err != nil {
return nil, err
}
// Collect commits visible from FETCH_HEAD that aren't visible from master.
commitsText, err := ctx.Git().Log("FETCH_HEAD", "master", "%an%n%ae%n%B")
if err != nil {
return nil, err
}
// Format those commits and add them to the results.
for _, commitText := range commitsText {
if got, want := len(commitText), 3; got < want {
return nil, fmt.Errorf("Unexpected length of %v: got %v, want at least %v", commitText, got, want)
}
cls = append(cls, CL{
Author: commitText[0],
Email: commitText[1],
Description: strings.Join(commitText[2:], "\n"),
})
}
default:
return nil, UnsupportedProtocolErr(updateOp.project.Protocol)
}
}
update[name] = cls
}
return update, nil
}
// hostCredentials returns credentials for the given Gerrit host. The
// function uses best effort to scan common locations where the
// credentials could exist.
func hostCredentials(run *runutil.Run, host string) (_ *credentials, e error) {
// Check the host URL is valid.
url, err := url.Parse(host)
if err != nil {
return nil, fmt.Errorf("Parse(%q) failed: %v", host, err)
}
if url.Host == "" {
return nil, fmt.Errorf("%q has no host", host)
}
// Look for the host credentials in the .netrc file.
netrcPath := filepath.Join(os.Getenv("HOME"), ".netrc")
file, err := run.Open(netrcPath)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
} else {
defer collect.Error(func() error { return file.Close() }, &e)
credsMap, err := parseNetrcFile(file)
if err != nil {
return nil, err
}
creds, ok := credsMap[url.Host]
if ok {
return creds, nil
}
}
// Look for the host credentials in the git cookie file.
args := []string{"config", "--get", "http.cookiefile"}
var stdout, stderr bytes.Buffer
opts := run.Opts()
opts.Stdout = &stdout
opts.Stderr = &stderr
if err := run.CommandWithOpts(opts, "git", args...); err == nil {
cookieFilePath := strings.TrimSpace(stdout.String())
file, err := run.Open(cookieFilePath)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
} else {
defer collect.Error(func() error { return file.Close() }, &e)
credsMap, err := parseGitCookieFile(file)
if err != nil {
return nil, err
}
creds, ok := credsMap[url.Host]
if ok {
return creds, nil
}
// Account for site-wide credentials. Namely, the git cookie
// file can contain credentials of the form ".<name>", which
// should match any host "*.<name>".
for host, creds := range credsMap {
if strings.HasPrefix(host, ".") && strings.HasSuffix(url.Host, host) {
return creds, nil
}
}
}
}
return nil, fmt.Errorf("cannot find credentials for %q", host)
}
func syncCL(ctx *tool.Context) (e error) {
stashed, err := ctx.Git().Stash()
if err != nil {
return err
}
if stashed {
defer collect.Error(func() error { return ctx.Git().StashPop() }, &e)
}
// Register a cleanup handler in case of subsequent errors.
forceOriginalBranch := true
originalBranch, err := ctx.Git().CurrentBranchName()
if err != nil {
return err
}
originalWd, err := os.Getwd()
if err != nil {
return err
}
defer func() {
if forceOriginalBranch {
ctx.Git().CheckoutBranch(originalBranch, gitutil.ForceOpt(true))
}
ctx.Run().Chdir(originalWd)
}()
// Switch to an existing directory in master so we can run commands.
topLevel, err := ctx.Git().TopLevel()
if err != nil {
return err
}
if err := ctx.Run().Chdir(topLevel); err != nil {
return err
}
// Identify the dependents CLs leading to (and including) the
// current branch.
branches, err := getDependentCLs(ctx, originalBranch)
if err != nil {
return err
}
branches = append(branches, originalBranch)
// Sync from upstream.
if err := ctx.Git().CheckoutBranch(branches[0]); err != nil {
return err
}
if err := ctx.Git().Pull("origin", branches[0]); err != nil {
return err
}
// Bring all CLs in the sequence of dependent CLs leading to the
// current branch up to date with the <remoteBranchFlag> branch.
for i := 1; i < len(branches); i++ {
if err := ctx.Git().CheckoutBranch(branches[i]); err != nil {
return err
}
if err := ctx.Git().Merge(branches[i-1]); err != nil {
return fmt.Errorf(`Failed to automatically merge branch %v into branch %v: %v
The following steps are needed before the operation can be retried:
$ git checkout %v
$ git merge %v
# resolve all conflicts
$ git commit -a
$ git checkout %v
# retry the original operation
`, branches[i], branches[i-1], err, branches[i], branches[i-1], originalBranch)
}
}
forceOriginalBranch = false
return nil
}
// createReviewBranch creates a clean review branch from the remote
// branch this CL pertains to and then iterates over the sequence of
// dependent CLs leading to the current branch, creating one commit
// per CL by squashing all commits of each individual CL. The commit
// message for all but that last CL is derived from their
// <commitMessageFileName>, while the <message> argument is used as
// the commit message for the last commit.
func (review *review) createReviewBranch(message string) (e error) {
// Create the review branch.
if err := review.ctx.Git().FetchRefspec("origin", review.CLOpts.RemoteBranch); err != nil {
return err
}
if review.ctx.Git().BranchExists(review.reviewBranch) {
if err := review.ctx.Git().DeleteBranch(review.reviewBranch, gitutil.ForceOpt(true)); err != nil {
return err
}
}
upstream := "origin/" + review.CLOpts.RemoteBranch
if err := review.ctx.Git().CreateBranchWithUpstream(review.reviewBranch, upstream); err != nil {
return err
}
if err := review.ctx.Git().CheckoutBranch(review.reviewBranch); err != nil {
return err
}
// Register a cleanup handler in case of subsequent errors.
cleanup := true
defer collect.Error(func() error {
if !cleanup {
return review.ctx.Git().CheckoutBranch(review.CLOpts.Branch)
}
review.ctx.Git().CheckoutBranch(review.CLOpts.Branch, gitutil.ForceOpt(true))
review.ctx.Git().DeleteBranch(review.reviewBranch, gitutil.ForceOpt(true))
return nil
}, &e)
// Report an error if the CL is empty.
if !review.ctx.DryRun() {
hasDiff, err := review.ctx.Git().BranchesDiffer(review.CLOpts.Branch, review.reviewBranch)
if err != nil {
return err
}
if !hasDiff {
return emptyChangeError(struct{}{})
}
}
// If <message> is empty, replace it with the default message.
if len(message) == 0 {
var err error
message, err = review.defaultCommitMessage()
if err != nil {
return err
}
}
// Iterate over all dependent CLs leading to (and including) the
// current branch, creating one commit in the review branch per CL
// by squashing all commits of each individual CL.
branches, err := getDependentCLs(review.ctx, review.CLOpts.Branch)
if err != nil {
return err
}
branches = append(branches, review.CLOpts.Branch)
if err := review.squashBranches(branches, message); err != nil {
return err
}
cleanup = false
return nil
}
// squashBranches iterates over the given list of branches, creating
// one commit per branch in the current branch by squashing all
// commits of each individual branch.
//
// TODO(jsimsa): Consider using "git rebase --onto" to avoid having to
// deal with merge conflicts.
func (review *review) squashBranches(branches []string, message string) (e error) {
for i := 1; i < len(branches); i++ {
// We want to merge the <branches[i]> branch on top of the review
// branch, forcing all conflicts to be reviewed in favor of the
// <branches[i]> branch. Unfortunately, git merge does not offer a
// strategy that would do that for us. The solution implemented
// here is based on:
//
// http://stackoverflow.com/questions/173919/is-there-a-theirs-version-of-git-merge-s-ours
if err := review.ctx.Git().Merge(branches[i], gitutil.SquashOpt(true), gitutil.StrategyOpt("ours")); err != nil {
return changeConflictError{
localBranch: branches[i],
remoteBranch: review.CLOpts.RemoteBranch,
message: err.Error(),
}
}
// Fetch the timestamp of the last commit of <branches[i]> and use
// it to create the squashed commit. This is needed to make sure
// that the commit hash of the squashed commit stays the same as
// long as the squashed sequence of commits does not change. If
// this was not the case, consecutive invocations of "jiri cl mail"
// could fail if some, but not all, of the dependent CLs submitted
// to Gerrit have changed.
output, err := review.ctx.Git().Log(branches[i], branches[i]+"^", "%ad%n%cd")
if err != nil {
return err
}
if len(output) < 1 || len(output[0]) < 2 {
return fmt.Errorf("unexpected output length: %v", output)
}
authorDate := tool.AuthorDateOpt(output[0][0])
committerDate := tool.CommitterDateOpt(output[0][1])
if i < len(branches)-1 {
file, err := getCommitMessageFileName(review.ctx, branches[i])
if err != nil {
return err
}
message, err := review.ctx.Run().ReadFile(file)
if err != nil {
return err
}
if err := review.ctx.Git(authorDate, committerDate).CommitWithMessage(string(message)); err != nil {
return err
}
} else {
committer := review.ctx.Git(authorDate, committerDate).NewCommitter(review.CLOpts.Edit)
if err := committer.Commit(message); err != nil {
return err
}
}
tmpBranch := review.reviewBranch + "-" + branches[i] + "-TMP"
if err := review.ctx.Git().CreateBranch(tmpBranch); err != nil {
return err
}
defer collect.Error(func() error {
return review.ctx.Git().DeleteBranch(tmpBranch, gitutil.ForceOpt(true))
}, &e)
if err := review.ctx.Git().Reset(branches[i]); err != nil {
return err
}
if err := review.ctx.Git().Reset(tmpBranch, gitutil.ModeOpt("soft")); err != nil {
return err
}
if err := review.ctx.Git(authorDate, committerDate).CommitAmend(); err != nil {
return err
}
}
return nil
}
// run implements checks that the review passes all local checks
// and then mails it to Gerrit.
func (review *review) run() (e error) {
if uncommittedFlag {
changes, err := review.ctx.Git().FilesWithUncommittedChanges()
if err != nil {
return err
}
if len(changes) != 0 {
return uncommittedChangesError(changes)
}
}
if review.CLOpts.Branch == remoteBranchFlag {
return fmt.Errorf("cannot do a review from the %q branch.", remoteBranchFlag)
}
stashed, err := review.ctx.Git().Stash()
if err != nil {
return err
}
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("Getwd() failed: %v", err)
}
defer collect.Error(func() error { return review.ctx.Run().Chdir(wd) }, &e)
topLevel, err := review.ctx.Git().TopLevel()
if err != nil {
return err
}
if err := review.ctx.Run().Chdir(topLevel); err != nil {
return err
}
defer collect.Error(func() error { return review.cleanup(stashed) }, &e)
file, err := getCommitMessageFileName(review.ctx, review.CLOpts.Branch)
if err != nil {
return err
}
message := messageFlag
if message == "" {
// Message was not passed in flag. Attempt to read it from file.
data, err := review.ctx.Run().ReadFile(file)
if err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
message = string(data)
}
}
// Add/remove labels to/from the commit message before asking users
// to edit it. We do this only when this is not the initial commit
// where the message is empty.
//
// For the initial commit, the labels will be processed after the
// message is edited by users, which happens in the
// updateReviewMessage method.
if message != "" {
message = review.processLabels(message)
}
if err := review.createReviewBranch(message); err != nil {
return err
}
if err := review.updateReviewMessage(file); err != nil {
return err
}
if err := review.send(); err != nil {
return err
}
if setTopicFlag {
if err := review.setTopic(); err != nil {
return err
}
}
return nil
}
func (op createOperation) Run(ctx *tool.Context, manifest *Manifest) (e error) {
hosts, _, _, _, err := readManifest(ctx, false)
if err != nil {
return err
}
path, perm := filepath.Dir(op.destination), os.FileMode(0755)
if err := ctx.Run().MkdirAll(path, perm); err != nil {
return err
}
// Create a temporary directory for the initial setup of the
// project to prevent an untimely termination from leaving the
// $JIRI_ROOT directory in an inconsistent state.
tmpDirPrefix := strings.Replace(op.Project().Name, "/", ".", -1) + "-"
tmpDir, err := ctx.Run().TempDir(path, tmpDirPrefix)
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
switch op.project.Protocol {
case "git":
if err := ctx.Git().Clone(op.project.Remote, tmpDir); err != nil {
return err
}
// Apply git hooks. We're creating this repo, so there's no danger of
// overriding existing hooks. Customizing your git hooks with jiri is a bad
// idea anyway, since jiri won't know to not delete the project when you
// switch between manifests or do a cleanup.
host, found := hosts["git"]
if found && strings.HasPrefix(op.project.Remote, host.Location) {
gitHookDir := filepath.Join(tmpDir, ".git", "hooks")
for _, githook := range host.GitHooks {
mdir, err := ManifestDir()
if err != nil {
return err
}
src, err := ctx.Run().ReadFile(filepath.Join(mdir, githook.Path))
if err != nil {
return err
}
dst := filepath.Join(gitHookDir, githook.Name)
if err := ctx.Run().WriteFile(dst, src, perm); err != nil {
return err
}
}
}
// Apply exclusion for /.jiri/. We're creating the repo so we can safely
// write to .git/info/exclude
excludeString := "/.jiri/\n"
excludeDir := filepath.Join(tmpDir, ".git", "info")
if err := ctx.Run().MkdirAll(excludeDir, os.FileMode(0750)); err != nil {
return err
}
excludeFile := filepath.Join(excludeDir, "exclude")
if err := ctx.Run().WriteFile(excludeFile, []byte(excludeString), perm); err != nil {
return err
}
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return ctx.Run().Chdir(cwd) }, &e)
if err := ctx.Run().Chdir(tmpDir); err != nil {
return err
}
if err := ctx.Git().Reset(op.project.Revision); err != nil {
return err
}
default:
return UnsupportedProtocolErr(op.project.Protocol)
}
if err := writeMetadata(ctx, op.project, tmpDir); err != nil {
return err
}
if err := ctx.Run().Chmod(tmpDir, os.FileMode(0755)); err != nil {
return err
}
if err := ctx.Run().Rename(tmpDir, op.destination); err != nil {
return err
}
if err := resetProject(ctx, op.project); err != nil {
return err
}
return addProjectToManifest(ctx, manifest, op.project)
}