Exemple #1
0
func main() {
	waitTime := time.Millisecond * 200
	p := uiprogress.New()
	p.Start()

	var wg sync.WaitGroup

	bar1 := p.AddBar(20).AppendCompleted().PrependElapsed()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for bar1.Incr() {
			time.Sleep(waitTime)
		}
		fmt.Fprintln(p.Bypass(), "Bar1 finished")
	}()

	bar2 := p.AddBar(40).AppendCompleted().PrependElapsed()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for bar2.Incr() {
			time.Sleep(waitTime)
		}
		fmt.Fprintln(p.Bypass(), "Bar2 finished")
	}()

	time.Sleep(time.Second)
	bar3 := p.AddBar(20).PrependElapsed().AppendCompleted()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for bar3.Incr() {
			time.Sleep(waitTime)
		}
		fmt.Fprintln(p.Bypass(), "Bar3 finished")
	}()

	wg.Wait()
}
Exemple #2
0
func NewFsDriver(bucket, username, password, curDir string, conc int,
	logger *log.Logger) (*FsDriver, error) {
	driver := &FsDriver{
		curDir:   curDir,
		operator: username,
		bucket:   bucket,
		up:       upyun.NewUpYun(bucket, username, password),
		maxConc:  conc,
		logger:   logger,
	}

	var err error
	_, err = driver.up.Usage()
	if err != nil {
		return nil, err
	}

	driver.progress = uiprogress.New()
	driver.progress.RefreshInterval = time.Millisecond * 100
	driver.progress.Start()

	return driver, nil
}
Exemple #3
0
// Executes the "push" command
func execPushCommand(c *cli.Context) error {
	if len(c.Args()) < 1 {
		return cli.NewExitError(errNoModuleName.Error(), 64)
	}

	// Create the task that we send to our minions
	// The task's command is the module name that will be
	// loaded and processed by the remote minions
	main := c.Args()[0]
	t := task.New(main, c.String("environment"))

	client := newEtcdMinionClientFromFlags(c)

	cFlag := c.String("with-classifier")
	minions, err := parseClassifierPattern(client, cFlag)

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	numMinions := len(minions)
	if numMinions == 0 {
		return cli.NewExitError(errNoMinionFound.Error(), 1)
	}

	fmt.Printf("Found %d minion(s) for task processing\n\n", numMinions)

	// Progress bar to display while submitting task
	progress := uiprogress.New()
	bar := progress.AddBar(numMinions)
	bar.AppendCompleted()
	bar.PrependElapsed()
	progress.Start()

	// Number of minions to which submitting the task has failed
	failed := 0

	// Submit task to minions
	fmt.Println("Submitting task to minion(s) ...")
	for _, m := range minions {
		err = client.MinionSubmitTask(m, t)
		if err != nil {
			fmt.Printf("Failed to submit task to %s: %s\n", m, err)
			failed++
		}
		bar.Incr()
	}

	// Stop progress bar and sleep for a bit to make sure the
	// progress bar gets updated if we were too fast for it
	progress.Stop()
	time.Sleep(time.Millisecond * 100)

	// Display task report
	fmt.Println()
	table := uitable.New()
	table.MaxColWidth = 80
	table.Wrap = true
	table.AddRow("TASK", "SUBMITTED", "FAILED", "TOTAL")
	table.AddRow(t.ID, numMinions-failed, failed, numMinions)
	fmt.Println(table)

	return nil
}
Exemple #4
0
// Executes the "run" command
func execRunCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errMissingTask, 64)
	}

	client := newEtcdMinionClientFromFlags(c)

	cFlag := c.String("with-classifier")
	minions, err := parseClassifierPattern(client, cFlag)

	if err != nil {
		displayError(err, 1)
	}

	numMinions := len(minions)
	if numMinions == 0 {
		displayError(errNoMinionFound, 1)
	}

	fmt.Printf("Found %d minion(s) for task processing\n\n", numMinions)

	// Create the task that we send to our minions
	// The first argument is the command and anything else
	// that follows is considered task arguments
	args := c.Args()
	isConcurrent := c.Bool("is-concurrent")
	taskCommand := args[0]
	taskArgs := args[1:]
	t := task.New(taskCommand, taskArgs...)
	t.IsConcurrent = isConcurrent

	// Progress bar to display while submitting task
	progress := uiprogress.New()
	bar := progress.AddBar(numMinions)
	bar.AppendCompleted()
	bar.PrependElapsed()
	progress.Start()

	// Number of minions to which submitting the task has failed
	failed := 0

	// Submit task to minions
	fmt.Println("Submitting task to minion(s) ...")
	for _, minion := range minions {
		err = client.MinionSubmitTask(minion, t)
		if err != nil {
			fmt.Printf("Failed to submit task to %s: %s\n", minion, err)
			failed += 1
		}
		bar.Incr()
	}

	// Stop progress bar and sleep for a bit to make sure the
	// progress bar gets updated if we were too fast for it
	progress.Stop()
	time.Sleep(time.Millisecond * 100)

	// Display task report
	fmt.Println()
	table := uitable.New()
	table.MaxColWidth = 80
	table.Wrap = true
	table.AddRow("TASK", "SUBMITTED", "FAILED", "TOTAL")
	table.AddRow(t.TaskID, numMinions-failed, failed, numMinions)
	fmt.Println(table)
}
Exemple #5
0
func main() {
	log.SetFlags(0)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage)
		flag.PrintDefaults()
	}
	flag.Parse()
	if *flRepoTag && *flTagRepo {
		fmt.Fprintln(os.Stderr, "-repo/tag is not compatible with -tag/repo")
		os.Exit(1)
	}
	mode := modes[*flMode]
	if mode == NoMode {
		fmt.Fprintln(os.Stderr, "Unknown mode: ", *flMode)
		os.Exit(1)
	}
	remotePaths := flag.Args()
	if len(remotePaths) < 1 {
		flag.Usage()
		os.Exit(1)
	}

	nSides := len(remotePaths)
	commitInfo := make(map[git.Oid]CommitInfo)
	finalBranchInfo := make(map[string][]BranchInfo)
	roots := []*git.Commit{}
	sideNames := make([]string, nSides)
	for i, path := range remotePaths {
		sideNames[i] = filepath.Base(path)
	}
	var sideNamesTree *Tree
	switch {
	case *flRepoTag:
		sideNamesTree = StringTree(sideNames)
	case *flTagRepo:
		sideNamesTree = StringTree(reverseStrings(sideNames))
	}

	repo, err := git.InitRepository(".", false)
	e.Exit(err)
	tagList, err := repo.Tags.List()
	e.Exit(err)
	if *flTagRepo {
		tagList = reverseStrings(tagList)
	}
	var relevantTagList []string

	for i, remotePath := range remotePaths {
		// Fetch.
		remotePath, err := filepath.Abs(remotePath)
		e.Exit(err)

		name := sideNames[i]
		remoteGlob := fmt.Sprint("refs/remotes/", name, "/*")
		switch {
		case *flRepoTag:
			sideNamesTree.Insert(name, true)
		case *flTagRepo:
			sideNamesTree.Insert(stringutil.Reverse(name), true)
		}

		remote, err := repo.Remotes.Lookup(name)
		if err != nil {
			log.Printf("fetching %q (%s)", name, err)
			remote, err = repo.Remotes.Create(name, remotePath)
			e.Exit(err)
			// Use "git fetch" because libgit2 fetch is horribly slow for big local repos.
			// err = remote.Fetch(nil, nil, "")
			_ = remote
			cmd := exec.Command("git", "fetch", name)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			err = cmd.Run()
			e.Exit(err)
		}

		// For each commit to be rewritten, store it side number in commitInfo.
		log.Printf("walking %q", name)
		walker, err := repo.Walk()
		e.Exit(err)
		err = walker.PushGlob(remoteGlob)
		e.Exit(err)
		if *flTagRepo || *flRepoTag {
			key := name
			if *flTagRepo {
				key = stringutil.Reverse(name)
			}
			relevantTags := sideNamesTree.FilterStrings(key, tagList)
			for _, tagName := range relevantTags {
				if *flTagRepo {
					tagName = stringutil.Reverse(tagName)
				}
				ref, err := repo.References.Lookup("refs/tags/" + tagName)
				e.Exit(err)
				obj, err := ref.Peel(git.ObjectCommit)
				if err != nil {
					// Ignore tags that do not point to commits.
					continue
				}
				relevantTagList = append(relevantTagList, tagName)
				walker.Push(obj.Id())
				// Add the tag to the list of roots for rewrite.
				commit, err := repo.LookupCommit(obj.Id())
				e.Exit(err)
				roots = append(roots, commit)
			}
		}
		sharedCommitsCount := 0
		err = walker.Iterate(func(commit *git.Commit) bool {
			oid := *commit.Id()
			if _, ok := commitInfo[oid]; !ok {
				commitInfo[oid] = CommitInfo{Side: i}
			} else {
				sharedCommitsCount++
			}
			return true
		})
		e.Exit(err)
		if sharedCommitsCount > 0 {
			log.Printf("did not change side of %s", plural(sharedCommitsCount, "shared commit"))
		}

		// Update commitInfo with lists of commit branches.
		iter, err := repo.NewReferenceIteratorGlob(remoteGlob)
		e.Exit(err)
		for {
			ref, err := iter.Next()
			if git.IsErrorCode(err, git.ErrIterOver) {
				break
			}
			e.Exit(err)
			// Seed search with head commit.
			refCommit, err := repo.LookupCommit(ref.Target())
			e.Exit(err)
			roots = append(roots, refCommit)
			// Store commit branches.
			branchName, err := ref.Branch().Name()
			e.Exit(err)
			branchName = strings.TrimPrefix(branchName, name+"/")
			if mode == FinalMode && branchName != *flInterpose {
				// Add branches other than "master" for final merge.
				finalBranchInfo[branchName] = append(finalBranchInfo[branchName], BranchInfo{
					Side:   i,
					Commit: refCommit,
				})
				// Skip interposing.
				continue
			}
			walker.Reset()
			err = walker.Push(ref.Target())
			e.Exit(err)
			walker.SimplifyFirstParent()
			err = walker.Iterate(func(commit *git.Commit) bool {
				oid := *commit.Id()
				ci := commitInfo[oid]
				ci.Branches = append(ci.Branches, branchName)
				if mode == TotalMode && branchName == "master" {
					// Reduce risk of missing sibling trees
					// with our strategy that takes them
					// from the first parent by making
					// "master" the first parent.
					n := len(ci.Branches) - 1
					ci.Branches[0], ci.Branches[n] = ci.Branches[n], ci.Branches[0]
				}

				commitInfo[oid] = ci
				return true
			})
			e.Exit(err)
		}
	}

	log.Printf("composing %s (%s)", plural(nSides, "side"), plural(len(commitInfo), "commit"))
	tb, err := repo.TreeBuilder()
	e.Exit(err)
	emptyTreeOid, err := tb.Write()
	e.Exit(err)
	emptyTree, err := repo.LookupTree(emptyTreeOid)
	e.Exit(err)
	sig := &git.Signature{Name: "root", Email: "root"} // Deterministic signature.
	composedOid, err := repo.CreateCommit("", sig, sig, "", emptyTree, roots...)
	e.Exit(err)
	if *flVerbose {
		log.Println("virtual common head:", composedOid)
	}

	filterMapping := make(map[git.Oid]git.Oid) // Map old commits to new commits.
	newHeads := make(map[string]*git.Commit)   // Map new heads to new commits.

	var progress *uiprogress.Progress
	var bar *uiprogress.Bar
	if !*flVerbose {
		progress = uiprogress.New()
		progress.Out = os.Stderr
		bar = progress.AddBar(len(commitInfo))
		progress.Start()
	}

	// libgit2 chronological topological walker in fact is not chronological.
	// totalWalker, err := repo.Walk()
	totalWalker, err := NewReverseTopologicalDateOrderCommitWalker(repo)
	e.Exit(err)
	// totalWalker.Sorting(git.SortTopological | git.SortTime | git.SortReverse)
	totalWalker.Push(composedOid)
	err = totalWalker.Iterate(func(commit *git.Commit) bool {
		oid := *commit.Id()
		if oid == *composedOid {
			return true // Skip our composed oid (and immediately finish).
		}
		ci := commitInfo[oid]

		parents := []*git.Commit{}
		useParentsFrom := uint(0)
		if len(ci.Branches) > 0 {
			// Replace first parent (if any) with heads of relevant branches.
			useParentsFrom = 1
			usedHeads := map[git.Oid]bool{}
			for _, branch := range ci.Branches {
				head := newHeads[branch]
				if head != nil && !usedHeads[*head.Id()] {
					parents = append(parents, head)
					usedHeads[*head.Id()] = true
				}
			}
		}
		// Translate old parents.
		for i := useParentsFrom; i < commit.ParentCount(); i++ {
			oldParentOid := commit.Parent(uint(i)).Id()
			parentOid := filterMapping[*oldParentOid]
			newParent, err := repo.LookupCommit(&parentOid)
			if err != nil {
				e.Exit(fmt.Errorf("Error: %v parent %v in %q was not mapped yet", &oid, oldParentOid, sideNames[ci.Side]))
			}
			parents = append(parents, newParent)
		}
		// Update trees.
		var tb *git.TreeBuilder
		if len(parents) > 0 {
			parentTree, err := parents[0].Tree()
			e.Exit(err)
			tb, err = repo.TreeBuilderFromTree(parentTree)
			e.Exit(err)
		} else {
			tb, err = repo.TreeBuilder()
			e.Exit(err)
		}
		tree, err := commit.Tree()
		e.Exit(err)
		err = tb.Insert(sideNames[ci.Side], tree.Id(), int(git.FilemodeTree))
		e.Exit(err)
		newTreeID, err := tb.Write()
		e.Exit(err)
		newTree, err := repo.LookupTree(newTreeID)
		e.Exit(err)
		// Commit.
		newOid, err := repo.CreateCommit("", commit.Author(), commit.Committer(), commit.Message(), newTree, parents...)
		e.Exit(err)
		filterMapping[oid] = *newOid
		// Update cached heads.
		newCommit, err := repo.LookupCommit(newOid)
		e.Exit(err)
		for _, branch := range ci.Branches {
			newHeads[branch] = newCommit
		}

		if bar != nil {
			bar.Incr()
		}
		if *flVerbose {
			fmt.Println(commit.Committer().When, &oid, ci.Side, ci.Branches, "→", newOid, len(parents))
		}

		return true
	})
	e.Exit(err)
	if progress != nil {
		progress.Stop()
	}

	// Create finally-merged branches.
	sig, err = repo.DefaultSignature()
	e.Exit(err)
	for branchName, branchInfos := range finalBranchInfo {
		nParents := len(branchInfos)
		var commitID *git.Oid
		if nParents == 1 {
			oldOid := filterMapping[*branchInfos[0].Commit.Id()]
			commit, err := repo.LookupCommit(&oldOid)
			e.Exit(err)
			commitID = commit.Id()
		} else {
			parents := make([]*git.Commit, nParents)
			parentNames := make([]string, nParents)
			tb, err := repo.TreeBuilder()
			e.Exit(err)
			for k, branchInfo := range branchInfos {
				parentOid := filterMapping[*branchInfo.Commit.Id()]
				parentCommit, err := repo.LookupCommit(&parentOid)
				e.Exit(err)
				parents[k] = parentCommit
				name := sideNames[branchInfo.Side]
				parentNames[k] = name
				oldTree, err := branchInfo.Commit.Tree()
				e.Exit(err)
				err = tb.Insert(name, oldTree.Id(), int(git.FilemodeTree))
				e.Exit(err)
			}
			newTreeID, err := tb.Write()
			e.Exit(err)
			newTree, err := repo.LookupTree(newTreeID)
			e.Exit(err)
			message := fmt.Sprintf("Compose branch '%s' from %s", branchName, strings.Join(parentNames, ", "))
			commitID, err = repo.CreateCommit("", sig, sig, message, newTree, parents...)
			e.Exit(err)
		}
		_, err = repo.References.Create("refs/heads/"+branchName, commitID, true, "")
		e.Exit(err)
	}
	// Create heads for interposed branches.
	for headName, commit := range newHeads {
		_, err = repo.References.Create("refs/heads/"+headName, commit.Id(), true, "")
		e.Exit(err)
	}
	// Create tags.
	rewrittenTagsCount := 0
	for {
		// This loop supports rewriting tags of tags...
		rewrittenTags := false
		for _, tagName := range relevantTagList {
			ref, err := repo.References.Lookup("refs/tags/" + tagName)
			e.Exit(err)
			oldOid := *ref.Target()
			if newOid, ok := filterMapping[oldOid]; ok {
				// Lightweight tag pointing to a rewritten commit.
				commit, err := repo.LookupCommit(&newOid)
				e.Exit(err)
				_, err = repo.Tags.CreateLightweight(tagName, commit, true)
				e.Exit(err)
				rewrittenTags = true
				rewrittenTagsCount++
				if *flVerbose {
					fmt.Println("rewritten lightweight tag", tagName, &oldOid, "→", &newOid)
				}
			}
			tag, err := repo.LookupTag(&oldOid)
			if err == nil {
				// Annotated tag.
				oldOid = *tag.Target().Id()
				if newOid, ok := filterMapping[oldOid]; ok {
					commit, err := repo.LookupCommit(&newOid)
					e.Exit(err)
					// Currently git2go does not support overwriting annotated tags.
					_, err = repo.Tags.Create(tagName, commit, tag.Tagger(), tag.Message(), true)
					e.Exit(err)
					rewrittenTags = true
					rewrittenTagsCount++
					if *flVerbose {
						fmt.Println("rewritten annotated tag", tagName, &oldOid, "→", &newOid)
					}
				}
			}
		}
		if !rewrittenTags {
			break
		}
	}
	log.Printf("merged %s, interposed %s, rewritten %s",
		plural(len(finalBranchInfo), "branch"),
		plural(len(newHeads), "branch"),
		plural(rewrittenTagsCount, "tag"))
}