// RateLimited returns a rate limited Notifier. only limit goroutines
// will be spawned. If limit is zero, no rate limiting happens. This
// is the same as `Notifier{}`.
func RateLimited(limit int) Notifier {
n := Notifier{}
if limit > 0 {
n.lim = ratelimit.NewRateLimiter(process.Background(), limit)
}
return n
}
func newNAT(realNAT nat.NAT) *NAT {
return &NAT{
nat: realNAT,
proc: goprocess.WithParent(goprocess.Background()),
mappings: make(map[*mapping]struct{}),
}
}
func newQueryRunner(q *dhtQuery) *dhtQueryRunner {
proc := process.WithParent(process.Background())
ctx := ctxproc.OnClosingContext(proc)
return &dhtQueryRunner{
query: q,
peersToQuery: queue.NewChanQueue(ctx, queue.NewXORDistancePQ(q.key)),
peersRemaining: todoctr.NewSyncCounter(),
peersSeen: pset.New(),
rateLimit: make(chan struct{}, q.concurrency),
proc: proc,
}
}
func TestRateLimitLimitedGoBlocks(t *testing.T) {
numChildren := 6
t.Logf("create a rate limiter with limit of %d", numChildren/2)
rl := NewRateLimiter(process.Background(), numChildren/2)
doneSpawning := make(chan struct{})
childClosing := make(chan struct{})
t.Log("spawn 6 children with LimitedGo.")
go func() {
for i := 0; i < numChildren; i++ {
rl.LimitedGo(func(child process.Process) {
// hang until we drain childClosing
childClosing <- struct{}{}
})
t.Logf("spawned %d", i)
}
close(doneSpawning)
}()
t.Log("should have blocked.")
select {
case <-doneSpawning:
t.Error("did not block")
case <-time.After(time.Millisecond): // for scheduler
t.Log("blocked")
}
t.Logf("drain %d children so they close", numChildren/2)
for i := 0; i < numChildren/2; i++ {
t.Logf("closing %d", i)
<-childClosing // consume child cloing
t.Logf("closed %d", i)
}
t.Log("should be done spawning.")
select {
case <-doneSpawning:
case <-time.After(100 * time.Millisecond): // for scheduler
t.Error("still blocked...")
}
t.Logf("drain %d children so they close", numChildren/2)
for i := 0; i < numChildren/2; i++ {
<-childClosing
t.Logf("closed %d", i)
}
rl.Close() // ensure everyone's closed.
}
func TestRateLimitGoDoesntBlock(t *testing.T) {
numChildren := 6
t.Logf("create a rate limiter with limit of %d", numChildren/2)
rl := NewRateLimiter(process.Background(), numChildren/2)
doneSpawning := make(chan struct{})
childClosing := make(chan struct{})
t.Log("spawn 6 children with usual Process.Go.")
go func() {
for i := 0; i < numChildren; i++ {
rl.Go(func(child process.Process) {
// hang until we drain childClosing
childClosing <- struct{}{}
})
t.Logf("spawned %d", i)
}
close(doneSpawning)
}()
t.Log("should not have blocked.")
select {
case <-doneSpawning:
t.Log("did not block")
case <-time.After(100 * time.Millisecond): // for scheduler
t.Error("process.Go blocked. it should not.")
}
t.Log("drain children so they close")
for i := 0; i < numChildren; i++ {
<-childClosing
t.Logf("closed %d", i)
}
rl.Close() // ensure everyone's closed.
}
func (s *Swarm) dialAddrs(ctx context.Context, d *conn.Dialer, p peer.ID, remoteAddrs []ma.Multiaddr) (conn.Conn, error) {
// try to connect to one of the peer's known addresses.
// we dial concurrently to each of the addresses, which:
// * makes the process faster overall
// * attempts to get the fastest connection available.
// * mitigates the waste of trying bad addresses
log.Debugf("%s swarm dialing %s %s", s.local, p, remoteAddrs)
ctx, cancel := context.WithCancel(ctx)
defer cancel() // cancel work when we exit func
foundConn := make(chan struct{})
conns := make(chan conn.Conn, len(remoteAddrs))
errs := make(chan error, len(remoteAddrs))
// dialSingleAddr is used in the rate-limited async thing below.
dialSingleAddr := func(addr ma.Multiaddr) {
connC, err := s.dialAddr(ctx, d, p, addr)
// check parent still wants our results
select {
case <-foundConn:
if connC != nil {
connC.Close()
}
return
default:
}
if err != nil {
errs <- err
} else if connC == nil {
errs <- fmt.Errorf("failed to dial %s %s", p, addr)
} else {
conns <- connC
}
}
// this whole thing is in a goroutine so we can use foundConn
// to end early.
go func() {
// rate limiting just in case. at most 10 addrs at once.
limiter := ratelimit.NewRateLimiter(process.Background(), 10)
limiter.Go(func(worker process.Process) {
// permute addrs so we try different sets first each time.
for _, i := range rand.Perm(len(remoteAddrs)) {
select {
case <-foundConn: // if one of them succeeded already
break
case <-worker.Closing(): // our context was cancelled
break
default:
}
workerAddr := remoteAddrs[i] // shadow variable to avoid race
limiter.LimitedGo(func(worker process.Process) {
dialSingleAddr(workerAddr)
})
}
})
processctx.CloseAfterContext(limiter, ctx)
}()
// wair fot the results.
exitErr := fmt.Errorf("failed to dial %s", p)
for i := 0; i < len(remoteAddrs); i++ {
select {
case exitErr = <-errs: //
log.Debug("dial error: ", exitErr)
case connC := <-conns:
// take the first + return asap
close(foundConn)
return connC, nil
}
}
return nil, exitErr
}
func run(ipfsPath, watchPath string) error {
proc := process.WithParent(process.Background())
log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath)
ipfsPath, err := homedir.Expand(ipfsPath)
if err != nil {
return err
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
defer watcher.Close()
if err := addTree(watcher, watchPath); err != nil {
return err
}
r, err := fsrepo.Open(ipfsPath)
if err != nil {
// TODO handle case: daemon running
// TODO handle case: repo doesn't exist or isn't initialized
return err
}
node, err := core.NewNode(context.Background(), &core.BuildCfg{
Online: true,
Repo: r,
})
if err != nil {
return err
}
defer node.Close()
if *http {
addr := "/ip4/127.0.0.1/tcp/5001"
var opts = []corehttp.ServeOption{
corehttp.GatewayOption(true),
corehttp.WebUIOption,
corehttp.CommandsOption(cmdCtx(node, ipfsPath)),
}
proc.Go(func(p process.Process) {
if err := corehttp.ListenAndServe(node, addr, opts...); err != nil {
return
}
})
}
interrupts := make(chan os.Signal)
signal.Notify(interrupts, os.Interrupt, os.Kill)
for {
select {
case <-interrupts:
return nil
case e := <-watcher.Events:
log.Printf("received event: %s", e)
isDir, err := IsDirectory(e.Name)
if err != nil {
continue
}
switch e.Op {
case fsnotify.Remove:
if isDir {
if err := watcher.Remove(e.Name); err != nil {
return err
}
}
default:
// all events except for Remove result in an IPFS.Add, but only
// directory creation triggers a new watch
switch e.Op {
case fsnotify.Create:
if isDir {
addTree(watcher, e.Name)
}
}
proc.Go(func(p process.Process) {
file, err := os.Open(e.Name)
if err != nil {
log.Println(err)
}
defer file.Close()
k, err := coreunix.Add(node, file)
if err != nil {
log.Println(err)
}
log.Printf("added %s... key: %s", e.Name, k)
})
}
case err := <-watcher.Errors:
log.Println(err)
}
}
return nil
}
// WithContext constructs and returns a Process that respects
// given context. It is the equivalent of:
//
// func ProcessWithContext(ctx context.Context) goprocess.Process {
// p := goprocess.WithParent(goprocess.Background())
// CloseAfterContext(p, ctx)
// return p
// }
//
func WithContext(ctx context.Context) goprocess.Process {
p := goprocess.WithParent(goprocess.Background())
CloseAfterContext(p, ctx)
return p
}