Beispiel #1
0
// RateLimited returns a rate limited Notifier. only limit goroutines
// will be spawned. If limit is zero, no rate limiting happens. This
// is the same as `Notifier{}`.
func RateLimited(limit int) Notifier {
	n := Notifier{}
	if limit > 0 {
		n.lim = ratelimit.NewRateLimiter(process.Background(), limit)
	}
	return n
}
Beispiel #2
0
func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}
Beispiel #3
0
func newQueryRunner(q *dhtQuery) *dhtQueryRunner {
	proc := process.WithParent(process.Background())
	ctx := ctxproc.OnClosingContext(proc)
	return &dhtQueryRunner{
		query:          q,
		peersToQuery:   queue.NewChanQueue(ctx, queue.NewXORDistancePQ(q.key)),
		peersRemaining: todoctr.NewSyncCounter(),
		peersSeen:      pset.New(),
		rateLimit:      make(chan struct{}, q.concurrency),
		proc:           proc,
	}
}
Beispiel #4
0
func TestRateLimitLimitedGoBlocks(t *testing.T) {
	numChildren := 6

	t.Logf("create a rate limiter with limit of %d", numChildren/2)
	rl := NewRateLimiter(process.Background(), numChildren/2)

	doneSpawning := make(chan struct{})
	childClosing := make(chan struct{})

	t.Log("spawn 6 children with LimitedGo.")
	go func() {
		for i := 0; i < numChildren; i++ {
			rl.LimitedGo(func(child process.Process) {
				// hang until we drain childClosing
				childClosing <- struct{}{}
			})
			t.Logf("spawned %d", i)
		}
		close(doneSpawning)
	}()

	t.Log("should have blocked.")
	select {
	case <-doneSpawning:
		t.Error("did not block")
	case <-time.After(time.Millisecond): // for scheduler
		t.Log("blocked")
	}

	t.Logf("drain %d children so they close", numChildren/2)
	for i := 0; i < numChildren/2; i++ {
		t.Logf("closing %d", i)
		<-childClosing // consume child cloing
		t.Logf("closed %d", i)
	}

	t.Log("should be done spawning.")
	select {
	case <-doneSpawning:
	case <-time.After(100 * time.Millisecond): // for scheduler
		t.Error("still blocked...")
	}

	t.Logf("drain %d children so they close", numChildren/2)
	for i := 0; i < numChildren/2; i++ {
		<-childClosing
		t.Logf("closed %d", i)
	}

	rl.Close() // ensure everyone's closed.
}
Beispiel #5
0
func TestRateLimitGoDoesntBlock(t *testing.T) {
	numChildren := 6

	t.Logf("create a rate limiter with limit of %d", numChildren/2)
	rl := NewRateLimiter(process.Background(), numChildren/2)

	doneSpawning := make(chan struct{})
	childClosing := make(chan struct{})

	t.Log("spawn 6 children with usual Process.Go.")
	go func() {
		for i := 0; i < numChildren; i++ {
			rl.Go(func(child process.Process) {
				// hang until we drain childClosing
				childClosing <- struct{}{}
			})
			t.Logf("spawned %d", i)
		}
		close(doneSpawning)
	}()

	t.Log("should not have blocked.")
	select {
	case <-doneSpawning:
		t.Log("did not block")
	case <-time.After(100 * time.Millisecond): // for scheduler
		t.Error("process.Go blocked. it should not.")
	}

	t.Log("drain children so they close")
	for i := 0; i < numChildren; i++ {
		<-childClosing
		t.Logf("closed %d", i)
	}

	rl.Close() // ensure everyone's closed.
}
Beispiel #6
0
func (s *Swarm) dialAddrs(ctx context.Context, d *conn.Dialer, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) {

	// try to connect to one of the peer's known addresses.
	// we dial concurrently to each of the addresses, which:
	// * makes the process faster overall
	// * attempts to get the fastest connection available.
	// * mitigates the waste of trying bad addresses
	log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel() // cancel work when we exit func

	foundConn := make(chan struct{})
	conns := make(chan conn.Conn, len(remoteAddrs))
	errs := make(chan error, len(remoteAddrs))

	// dialSingleAddr is used in the rate-limited async thing below.
	dialSingleAddr := func(addr ma.Multiaddr) {
		connC, err := s.dialAddr(ctx, d, p, addr)

		// check parent still wants our results
		select {
		case <-foundConn:
			if connC != nil {
				connC.Close()
			}
			return
		default:
		}

		if err != nil {
			errs <- err
		} else if connC == nil {
			errs <- fmt.Errorf("failed to dial %s %s", p, addr)
		} else {
			conns <- connC
		}
	}

	// this whole thing is in a goroutine so we can use foundConn
	// to end early.
	go func() {
		// rate limiting just in case. at most 10 addrs at once.
		limiter := ratelimit.NewRateLimiter(process.Background(), 10)
		limiter.Go(func(worker process.Process) {
			// permute addrs so we try different sets first each time.
			for _, i := range rand.Perm(len(remoteAddrs)) {
				select {
				case <-foundConn: // if one of them succeeded already
					break
				case <-worker.Closing(): // our context was cancelled
					break
				default:
				}

				workerAddr := remoteAddrs[i] // shadow variable to avoid race
				limiter.LimitedGo(func(worker process.Process) {
					dialSingleAddr(workerAddr)
				})
			}
		})

		processctx.CloseAfterContext(limiter, ctx)
	}()

	// wair fot the results.
	exitErr := fmt.Errorf("failed to dial %s", p)
	for i := 0; i < len(remoteAddrs); i++ {
		select {
		case exitErr = <-errs: //
			log.Debug("dial error: ", exitErr)
		case connC := <-conns:
			// take the first + return asap
			close(foundConn)
			return connC, nil
		}
	}
	return nil, exitErr
}
Beispiel #7
0
func run(ipfsPath, watchPath string) error {

	proc := process.WithParent(process.Background())
	log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath)

	ipfsPath, err := homedir.Expand(ipfsPath)
	if err != nil {
		return err
	}
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	if err := addTree(watcher, watchPath); err != nil {
		return err
	}

	r, err := fsrepo.Open(ipfsPath)
	if err != nil {
		// TODO handle case: daemon running
		// TODO handle case: repo doesn't exist or isn't initialized
		return err
	}

	node, err := core.NewNode(context.Background(), &core.BuildCfg{
		Online: true,
		Repo:   r,
	})
	if err != nil {
		return err
	}
	defer node.Close()

	if *http {
		addr := "/ip4/127.0.0.1/tcp/5001"
		var opts = []corehttp.ServeOption{
			corehttp.GatewayOption(true),
			corehttp.WebUIOption,
			corehttp.CommandsOption(cmdCtx(node, ipfsPath)),
		}
		proc.Go(func(p process.Process) {
			if err := corehttp.ListenAndServe(node, addr, opts...); err != nil {
				return
			}
		})
	}

	interrupts := make(chan os.Signal)
	signal.Notify(interrupts, os.Interrupt, os.Kill)

	for {
		select {
		case <-interrupts:
			return nil
		case e := <-watcher.Events:
			log.Printf("received event: %s", e)
			isDir, err := IsDirectory(e.Name)
			if err != nil {
				continue
			}
			switch e.Op {
			case fsnotify.Remove:
				if isDir {
					if err := watcher.Remove(e.Name); err != nil {
						return err
					}
				}
			default:
				// all events except for Remove result in an IPFS.Add, but only
				// directory creation triggers a new watch
				switch e.Op {
				case fsnotify.Create:
					if isDir {
						addTree(watcher, e.Name)
					}
				}
				proc.Go(func(p process.Process) {
					file, err := os.Open(e.Name)
					if err != nil {
						log.Println(err)
					}
					defer file.Close()
					k, err := coreunix.Add(node, file)
					if err != nil {
						log.Println(err)
					}
					log.Printf("added %s... key: %s", e.Name, k)
				})
			}
		case err := <-watcher.Errors:
			log.Println(err)
		}
	}
	return nil
}
Beispiel #8
0
// WithContext constructs and returns a Process that respects
// given context. It is the equivalent of:
//
//   func ProcessWithContext(ctx context.Context) goprocess.Process {
//     p := goprocess.WithParent(goprocess.Background())
//     CloseAfterContext(p, ctx)
//     return p
//   }
//
func WithContext(ctx context.Context) goprocess.Process {
	p := goprocess.WithParent(goprocess.Background())
	CloseAfterContext(p, ctx)
	return p
}