// RateLimited returns a rate limited Notifier. only limit goroutines // will be spawned. If limit is zero, no rate limiting happens. This // is the same as `Notifier{}`. func RateLimited(limit int) Notifier { n := Notifier{} if limit > 0 { n.lim = ratelimit.NewRateLimiter(process.Background(), limit) } return n }
func newNAT(realNAT nat.NAT) *NAT { return &NAT{ nat: realNAT, proc: goprocess.WithParent(goprocess.Background()), mappings: make(map[*mapping]struct{}), } }
func newQueryRunner(q *dhtQuery) *dhtQueryRunner { proc := process.WithParent(process.Background()) ctx := ctxproc.OnClosingContext(proc) return &dhtQueryRunner{ query: q, peersToQuery: queue.NewChanQueue(ctx, queue.NewXORDistancePQ(q.key)), peersRemaining: todoctr.NewSyncCounter(), peersSeen: pset.New(), rateLimit: make(chan struct{}, q.concurrency), proc: proc, } }
func TestRateLimitLimitedGoBlocks(t *testing.T) { numChildren := 6 t.Logf("create a rate limiter with limit of %d", numChildren/2) rl := NewRateLimiter(process.Background(), numChildren/2) doneSpawning := make(chan struct{}) childClosing := make(chan struct{}) t.Log("spawn 6 children with LimitedGo.") go func() { for i := 0; i < numChildren; i++ { rl.LimitedGo(func(child process.Process) { // hang until we drain childClosing childClosing <- struct{}{} }) t.Logf("spawned %d", i) } close(doneSpawning) }() t.Log("should have blocked.") select { case <-doneSpawning: t.Error("did not block") case <-time.After(time.Millisecond): // for scheduler t.Log("blocked") } t.Logf("drain %d children so they close", numChildren/2) for i := 0; i < numChildren/2; i++ { t.Logf("closing %d", i) <-childClosing // consume child cloing t.Logf("closed %d", i) } t.Log("should be done spawning.") select { case <-doneSpawning: case <-time.After(100 * time.Millisecond): // for scheduler t.Error("still blocked...") } t.Logf("drain %d children so they close", numChildren/2) for i := 0; i < numChildren/2; i++ { <-childClosing t.Logf("closed %d", i) } rl.Close() // ensure everyone's closed. }
func TestRateLimitGoDoesntBlock(t *testing.T) { numChildren := 6 t.Logf("create a rate limiter with limit of %d", numChildren/2) rl := NewRateLimiter(process.Background(), numChildren/2) doneSpawning := make(chan struct{}) childClosing := make(chan struct{}) t.Log("spawn 6 children with usual Process.Go.") go func() { for i := 0; i < numChildren; i++ { rl.Go(func(child process.Process) { // hang until we drain childClosing childClosing <- struct{}{} }) t.Logf("spawned %d", i) } close(doneSpawning) }() t.Log("should not have blocked.") select { case <-doneSpawning: t.Log("did not block") case <-time.After(100 * time.Millisecond): // for scheduler t.Error("process.Go blocked. it should not.") } t.Log("drain children so they close") for i := 0; i < numChildren; i++ { <-childClosing t.Logf("closed %d", i) } rl.Close() // ensure everyone's closed. }
func (s *Swarm) dialAddrs(ctx context.Context, d *conn.Dialer, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) { // try to connect to one of the peer's known addresses. // we dial concurrently to each of the addresses, which: // * makes the process faster overall // * attempts to get the fastest connection available. // * mitigates the waste of trying bad addresses log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs) ctx, cancel := context.WithCancel(ctx) defer cancel() // cancel work when we exit func foundConn := make(chan struct{}) conns := make(chan conn.Conn, len(remoteAddrs)) errs := make(chan error, len(remoteAddrs)) // dialSingleAddr is used in the rate-limited async thing below. dialSingleAddr := func(addr ma.Multiaddr) { connC, err := s.dialAddr(ctx, d, p, addr) // check parent still wants our results select { case <-foundConn: if connC != nil { connC.Close() } return default: } if err != nil { errs <- err } else if connC == nil { errs <- fmt.Errorf("failed to dial %s %s", p, addr) } else { conns <- connC } } // this whole thing is in a goroutine so we can use foundConn // to end early. go func() { // rate limiting just in case. at most 10 addrs at once. limiter := ratelimit.NewRateLimiter(process.Background(), 10) limiter.Go(func(worker process.Process) { // permute addrs so we try different sets first each time. for _, i := range rand.Perm(len(remoteAddrs)) { select { case <-foundConn: // if one of them succeeded already break case <-worker.Closing(): // our context was cancelled break default: } workerAddr := remoteAddrs[i] // shadow variable to avoid race limiter.LimitedGo(func(worker process.Process) { dialSingleAddr(workerAddr) }) } }) processctx.CloseAfterContext(limiter, ctx) }() // wair fot the results. exitErr := fmt.Errorf("failed to dial %s", p) for i := 0; i < len(remoteAddrs); i++ { select { case exitErr = <-errs: // log.Debug("dial error: ", exitErr) case connC := <-conns: // take the first + return asap close(foundConn) return connC, nil } } return nil, exitErr }
func run(ipfsPath, watchPath string) error { proc := process.WithParent(process.Background()) log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath) ipfsPath, err := homedir.Expand(ipfsPath) if err != nil { return err } watcher, err := fsnotify.NewWatcher() if err != nil { return err } defer watcher.Close() if err := addTree(watcher, watchPath); err != nil { return err } r, err := fsrepo.Open(ipfsPath) if err != nil { // TODO handle case: daemon running // TODO handle case: repo doesn't exist or isn't initialized return err } node, err := core.NewNode(context.Background(), &core.BuildCfg{ Online: true, Repo: r, }) if err != nil { return err } defer node.Close() if *http { addr := "/ip4/127.0.0.1/tcp/5001" var opts = []corehttp.ServeOption{ corehttp.GatewayOption(true), corehttp.WebUIOption, corehttp.CommandsOption(cmdCtx(node, ipfsPath)), } proc.Go(func(p process.Process) { if err := corehttp.ListenAndServe(node, addr, opts...); err != nil { return } }) } interrupts := make(chan os.Signal) signal.Notify(interrupts, os.Interrupt, os.Kill) for { select { case <-interrupts: return nil case e := <-watcher.Events: log.Printf("received event: %s", e) isDir, err := IsDirectory(e.Name) if err != nil { continue } switch e.Op { case fsnotify.Remove: if isDir { if err := watcher.Remove(e.Name); err != nil { return err } } default: // all events except for Remove result in an IPFS.Add, but only // directory creation triggers a new watch switch e.Op { case fsnotify.Create: if isDir { addTree(watcher, e.Name) } } proc.Go(func(p process.Process) { file, err := os.Open(e.Name) if err != nil { log.Println(err) } defer file.Close() k, err := coreunix.Add(node, file) if err != nil { log.Println(err) } log.Printf("added %s... key: %s", e.Name, k) }) } case err := <-watcher.Errors: log.Println(err) } } return nil }
// WithContext constructs and returns a Process that respects // given context. It is the equivalent of: // // func ProcessWithContext(ctx context.Context) goprocess.Process { // p := goprocess.WithParent(goprocess.Background()) // CloseAfterContext(p, ctx) // return p // } // func WithContext(ctx context.Context) goprocess.Process { p := goprocess.WithParent(goprocess.Background()) CloseAfterContext(p, ctx) return p }