func BenchmarkRunSingle(b *stdtesting.B) {
for i := 0; i < b.N; i++ {
r := parallel.NewRun(1)
r.Do(nothing)
r.Wait()
}
}
func (*parallelSuite) TestZeroWorkerPanics(c *gc.C) {
defer func() {
r := recover()
c.Check(r, gc.Matches, "parameter max must be >= 1")
}()
parallel.NewRun(0)
}
func (*parallelSuite) TestParallelError(c *gc.C) {
const (
totalDo = 10
errDo = 5
)
parallelRun := parallel.NewRun(6)
for i := 0; i < totalDo; i++ {
i := i
if i >= errDo {
parallelRun.Do(func() error {
return intError(i)
})
} else {
parallelRun.Do(func() error {
return nil
})
}
}
err := parallelRun.Wait()
c.Check(err, gc.NotNil)
errs := err.(parallel.Errors)
c.Check(len(errs), gc.Equals, totalDo-errDo)
ints := make([]int, len(errs))
for i, err := range errs {
ints[i] = int(err.(intError))
}
sort.Ints(ints)
for i, n := range ints {
c.Check(n, gc.Equals, i+errDo)
}
}
func fetchCharms(csRepo charmrepo.Interface, ids []string) (map[string]charm.Charm, error) {
charms := make([]charm.Charm, len(ids))
run := parallel.NewRun(30)
for i, id := range ids {
i, id := i, id
run.Do(func() error {
cref, err := charm.ParseReference(id)
if err != nil {
return errgo.Notef(err, "bad charm URL %q", id)
}
curl, err := csRepo.Resolve(cref)
if err != nil {
return errgo.Notef(err, "cannot resolve URL %q", id)
}
c, err := csRepo.Get(curl)
if err != nil {
return errgo.Notef(err, "cannot get %q", id)
}
charms[i] = c
return nil
})
}
if err := run.Wait(); err != nil {
return nil, err
}
m := make(map[string]charm.Charm)
for i, id := range ids {
m[id] = charms[i]
}
return m, nil
}
func BenchmarkRun1000p100(b *stdtesting.B) {
for i := 0; i < b.N; i++ {
r := parallel.NewRun(100)
for j := 0; j < 1000; j++ {
r.Do(nothing)
}
r.Wait()
}
}
func main() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "usage: check-mainline package...\n")
fmt.Fprintf(os.Stderr, `
This command checks that all dependencies of the named
packages are at a commit that is included in the origin remote.
Non-git repositories are ignored (with a warning).
`)
os.Exit(2)
}
flag.Parse()
pkgs := flag.Args()
cmd := exec.Command("godeps", append([]string{"-t"}, pkgs...)...)
cmd.Stderr = os.Stderr
out, err := cmd.Output()
if err != nil {
log.Fatal(err)
}
infos, err := parseDeps(bytes.NewReader(out))
if err != nil {
log.Fatal(err)
}
run := parallel.NewRun(10)
for _, info := range infos {
info := info
if info.vcs != "git" {
warningf("ignoring %s repo: %s", info.vcs, info.project)
continue
}
pkg, _ := build.Import(info.project, "", build.FindOnly)
if pkg.Dir == "" {
warningf("cannot find %s", info.project)
exitStatus = 1
continue
}
run.Do(func() error {
ok, err := inMainline(pkg.Dir, info.revid)
if err != nil {
return errgo.Notef(err, "warning: cannot determine mainline status for %s", info.project)
}
if !ok {
return errgo.Newf("%s is not mainline", info.project)
}
return nil
})
}
if err := run.Wait(); err != nil {
for _, e := range err.(parallel.Errors) {
fmt.Fprintln(os.Stderr, e)
}
os.Exit(1)
}
}
func (*parallelSuite) TestConcurrentDo(c *gc.C) {
r := parallel.NewRun(3)
var count int32
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
r.Do(func() error {
atomic.AddInt32(&count, 1)
return nil
})
wg.Done()
}()
}
wg.Wait()
err := r.Wait()
c.Assert(err, gc.IsNil)
c.Assert(count, gc.Equals, int32(100))
}
// FetchIcons retrieves icon SVGs over HTTP. If specified in the struct, icons
// will be fetched concurrently.
func (h *HTTPFetcher) FetchIcons(b *charm.BundleData) (map[string][]byte, error) {
client := http.DefaultClient
if h.Client != nil {
client = h.Client
}
concurrency := h.Concurrency
if concurrency <= 0 {
concurrency = 10
}
var iconsMu sync.Mutex // Guards icons.
icons := make(map[string][]byte)
alreadyFetched := make(map[string]bool)
run := parallel.NewRun(concurrency)
for _, serviceData := range b.Services {
charmId, err := charm.ParseReference(serviceData.Charm)
if err != nil {
return nil, errgo.Notef(err, "cannot parse charm %q", serviceData.Charm)
}
path := charmId.Path()
if alreadyFetched[path] {
continue
}
alreadyFetched[path] = true
run.Do(func() error {
icon, err := h.fetchIcon(h.IconURL(charmId), client)
if err != nil {
return err
}
iconsMu.Lock()
defer iconsMu.Unlock()
icons[path] = icon
return nil
})
}
if err := run.Wait(); err != nil {
return nil, err
}
return icons, nil
}
func (*parallelSuite) TestParallelMaxPar(c *gc.C) {
const (
totalDo = 10
maxConcurrentRunnersPar = 3
)
var mu sync.Mutex
maxConcurrentRunners := 0
nbRunners := 0
nbRuns := 0
parallelRunner := parallel.NewRun(maxConcurrentRunnersPar)
for i := 0; i < totalDo; i++ {
parallelRunner.Do(func() error {
mu.Lock()
nbRuns++
nbRunners++
if nbRunners > maxConcurrentRunners {
maxConcurrentRunners = nbRunners
}
mu.Unlock()
time.Sleep(time.Second / 10)
mu.Lock()
nbRunners--
mu.Unlock()
return nil
})
}
err := parallelRunner.Wait()
if nbRunners != 0 {
c.Errorf("%d functions still running", nbRunners)
}
if nbRuns != totalDo {
c.Errorf("all functions not executed; want %d got %d", totalDo, nbRuns)
}
c.Check(err, gc.IsNil)
if maxConcurrentRunners != maxConcurrentRunnersPar {
c.Errorf("wrong number of do's ran at once; want %d got %d", maxConcurrentRunnersPar, maxConcurrentRunners)
}
}
func delgroup(c cmd, conn *ec2.EC2, args []string) {
run := parallel.NewRun(40)
for _, g := range args {
g := g
run.Do(func() error {
var ec2g ec2.SecurityGroup
if secGroupPat.MatchString(g) {
ec2g.Id = g
} else {
ec2g.Name = g
}
_, err := conn.DeleteSecurityGroup(ec2g)
if err != nil {
errorf("cannot delete %q: %v", g, err)
return errgo.Newf("error")
}
return nil
})
}
if run.Wait() != nil {
os.Exit(1)
}
}