func (bs *Bitswap) rebroadcastWorker(parent context.Context) { ctx, cancel := context.WithCancel(parent) defer cancel() broadcastSignal := time.NewTicker(rebroadcastDelay.Get()) defer broadcastSignal.Stop() tick := time.NewTicker(10 * time.Second) defer tick.Stop() for { log.Event(ctx, "Bitswap.Rebroadcast.idle") select { case <-tick.C: n := bs.wm.wl.Len() if n > 0 { log.Debug(n, "keys in bitswap wantlist") } case <-broadcastSignal.C: // resend unfulfilled wantlist keys log.Event(ctx, "Bitswap.Rebroadcast.active") entries := bs.wm.wl.Entries() if len(entries) > 0 { bs.connectToProviders(ctx, entries) } case <-parent.Done(): return } } }
// GetBlocks gets a list of blocks asynchronously and returns through // the returned channel. // NB: No guarantees are made about order. func (s *BlockService) GetBlocks(ctx context.Context, ks []key.Key) <-chan *blocks.Block { out := make(chan *blocks.Block, 0) go func() { defer close(out) var misses []key.Key for _, k := range ks { hit, err := s.Blockstore.Get(k) if err != nil { misses = append(misses, k) continue } log.Debug("Blockservice: Got data in datastore.") select { case out <- hit: case <-ctx.Done(): return } } rblocks, err := s.Exchange.GetBlocks(ctx, misses) if err != nil { log.Debugf("Error with GetBlocks: %s", err) return } for b := range rblocks { select { case out <- b: case <-ctx.Done(): return } } }() return out }
func readMsgCtx(ctx context.Context, r msgio.Reader, p proto.Message) ([]byte, error) { var msg []byte // read in a goroutine so we can exit when our context is cancelled. done := make(chan error) go func() { var err error msg, err = r.ReadMsg() select { case done <- err: case <-ctx.Done(): } }() select { case <-ctx.Done(): return nil, ctx.Err() case e := <-done: if e != nil { return nil, e } } return msg, proto.Unmarshal(msg, p) }
// GetBlocks returns a channel where the caller may receive blocks that // correspond to the provided |keys|. Returns an error if BitSwap is unable to // begin this request within the deadline enforced by the context. // // NB: Your request remains open until the context expires. To conserve // resources, provide a context with a reasonably short deadline (ie. not one // that lasts throughout the lifetime of the server) func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan *blocks.Block, error) { select { case <-bs.process.Closing(): return nil, errors.New("bitswap is closed") default: } promise := bs.notifications.Subscribe(ctx, keys...) for _, k := range keys { log.Event(ctx, "Bitswap.GetBlockRequest.Start", &k) } bs.wm.WantBlocks(keys) req := &blockRequest{ keys: keys, ctx: ctx, } select { case bs.findKeys <- req: return promise, nil case <-ctx.Done(): return nil, ctx.Err() } }
func (cq *ChanQueue) process(ctx context.Context) { // construct the channels here to be able to use them bidirectionally enqChan := make(chan peer.ID) deqChan := make(chan peer.ID) cq.EnqChan = enqChan cq.DeqChan = deqChan go func() { log.Debug("processing") defer log.Debug("closed") defer close(deqChan) var next peer.ID var item peer.ID var more bool for { if cq.Queue.Len() == 0 { // log.Debug("wait for enqueue") select { case next, more = <-enqChan: if !more { return } // log.Debug("got", next) case <-ctx.Done(): return } } else { next = cq.Queue.Dequeue() // log.Debug("peek", next) } select { case item, more = <-enqChan: if !more { if cq.Queue.Len() > 0 { return // we're done done. } enqChan = nil // closed, so no use. } // log.Debug("got", item) cq.Queue.Enqueue(item) cq.Queue.Enqueue(next) // order may have changed. next = "" case deqChan <- next: // log.Debug("dequeued", next) next = "" case <-ctx.Done(): return } } }() }
func (bs *Bitswap) provideCollector(ctx context.Context) { defer close(bs.provideKeys) var toProvide []key.Key var nextKey key.Key var keysOut chan key.Key for { select { case blk, ok := <-bs.newBlocks: if !ok { log.Debug("newBlocks channel closed") return } if keysOut == nil { nextKey = blk.Key() keysOut = bs.provideKeys } else { toProvide = append(toProvide, blk.Key()) } case keysOut <- nextKey: if len(toProvide) > 0 { nextKey = toProvide[0] toProvide = toProvide[1:] } else { keysOut = nil } case <-ctx.Done(): return } } }
// Subscribe returns a channel of blocks for the given |keys|. |blockChannel| // is closed if the |ctx| times out or is cancelled, or after sending len(keys) // blocks. func (ps *impl) Subscribe(ctx context.Context, keys ...key.Key) <-chan *blocks.Block { blocksCh := make(chan *blocks.Block, len(keys)) valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking if len(keys) == 0 { close(blocksCh) return blocksCh } ps.wrapped.AddSubOnceEach(valuesCh, toStrings(keys)...) go func() { defer close(blocksCh) defer ps.wrapped.Unsub(valuesCh) // with a len(keys) buffer, this is an optimization for { select { case <-ctx.Done(): return case val, ok := <-valuesCh: if !ok { return } block, ok := val.(*blocks.Block) if !ok { return } select { case <-ctx.Done(): return case blocksCh <- block: // continue } } } }() return blocksCh }
// connects to providers for the given keys func (bs *Bitswap) providerConnector(parent context.Context) { defer log.Info("bitswap client worker shutting down...") for { log.Event(parent, "Bitswap.ProviderConnector.Loop") select { case req := <-bs.findKeys: keys := req.keys if len(keys) == 0 { log.Warning("Received batch request for zero blocks") continue } log.Event(parent, "Bitswap.ProviderConnector.Work", eventlog.LoggableMap{"Keys": keys}) // NB: Optimization. Assumes that providers of key[0] are likely to // be able to provide for all keys. This currently holds true in most // every situation. Later, this assumption may not hold as true. child, cancel := context.WithTimeout(req.ctx, providerRequestTimeout) providers := bs.network.FindProvidersAsync(child, keys[0], maxProvidersPerRequest) for p := range providers { go bs.network.ConnectTo(req.ctx, p) } cancel() case <-parent.Done(): return } } }
func (e *offlineExchange) GetBlocks(ctx context.Context, ks []key.Key) (<-chan *blocks.Block, error) { out := make(chan *blocks.Block, 0) go func() { defer close(out) var misses []key.Key for _, k := range ks { hit, err := e.bs.Get(k) if err != nil { misses = append(misses, k) // a long line of misses should abort when context is cancelled. select { // TODO case send misses down channel case <-ctx.Done(): return default: continue } } select { case out <- hit: case <-ctx.Done(): return } } }() return out, nil }
// gatedDialAttempt is an attempt to dial a node. It is gated by the swarm's // dial synchronization systems: dialsync and dialbackoff. func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error) { var logdial = lgbl.Dial("swarm", s.LocalPeer(), p, nil, nil) defer log.EventBegin(ctx, "swarmDialAttemptSync", logdial).Done() // check if we already have an open connection first conn := s.bestConnectionToPeer(p) if conn != nil { return conn, nil } // check if there's an ongoing dial to this peer if ok, wait := s.dsync.Lock(p); ok { // ok, we have been charged to dial! let's do it. // if it succeeds, dial will add the conn to the swarm itself. defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done() ctxT, cancel := context.WithTimeout(ctx, s.dialT) conn, err := s.dial(ctxT, p) cancel() s.dsync.Unlock(p) log.Debugf("dial end %s", conn) if err != nil { log.Event(ctx, "swarmDialBackoffAdd", logdial) s.backf.AddBackoff(p) // let others know to backoff // ok, we failed. try again. (if loop is done, our error is output) return nil, fmt.Errorf("dial attempt failed: %s", err) } log.Event(ctx, "swarmDialBackoffClear", logdial) s.backf.Clear(p) // okay, no longer need to backoff return conn, nil } else { // we did not dial. we must wait for someone else to dial. // check whether we should backoff first... if s.backf.Backoff(p) { log.Event(ctx, "swarmDialBackoff", logdial) return nil, ErrDialBackoff } defer log.EventBegin(ctx, "swarmDialWait", logdial).Done() select { case <-wait: // wait for that other dial to finish. // see if it worked, OR we got an incoming dial in the meantime... conn := s.bestConnectionToPeer(p) if conn != nil { return conn, nil } return nil, ErrDialFailed case <-ctx.Done(): // or we may have to bail... return nil, ctx.Err() } } }
func (pm *ProviderManager) AddProvider(ctx context.Context, k key.Key, val peer.ID) { prov := &addProv{ k: k, val: val, } select { case pm.newprovs <- prov: case <-ctx.Done(): } }
// WithContextAndTeardown is a helper function to set teardown at initiation // of WithContext func WithContextAndTeardown(ctx context.Context, tf goprocess.TeardownFunc) goprocess.Process { if ctx == nil { panic("nil Context") } p := goprocess.WithTeardown(tf) go func() { <-ctx.Done() p.Close() }() return p }
func (mq *msgQueue) runQueue(ctx context.Context) { for { select { case <-mq.work: // there is work to be done mq.doWork(ctx) case <-mq.done: return case <-ctx.Done(): return } } }
// CloseAfterContext schedules the process to close after the given // context is done. It is the equivalent of: // // func CloseAfterContext(p goprocess.Process, ctx context.Context) { // go func() { // <-ctx.Done() // p.Close() // }() // } // func CloseAfterContext(p goprocess.Process, ctx context.Context) { if p == nil { panic("nil Process") } if ctx == nil { panic("nil Context") } go func() { <-ctx.Done() p.Close() }() }
func PublishQueryEvent(ctx context.Context, ev *QueryEvent) { ich := ctx.Value(RoutingQueryKey) if ich == nil { return } ch, ok := ich.(chan<- *QueryEvent) if !ok { return } select { case ch <- ev: case <-ctx.Done(): } }
func (c *client) FindProvidersAsync(ctx context.Context, k key.Key, max int) <-chan peer.PeerInfo { out := make(chan peer.PeerInfo) go func() { defer close(out) for i, p := range c.server.Providers(k) { if max <= i { return } select { case out <- p: case <-ctx.Done(): return } } }() return out }
func (pm *ProviderManager) GetProviders(ctx context.Context, k key.Key) []peer.ID { gp := &getProv{ k: k, resp: make(chan []peer.ID, 1), // buffered to prevent sender from blocking } select { case <-ctx.Done(): return nil case pm.getprovs <- gp: } select { case <-ctx.Done(): return nil case peers := <-gp.resp: return peers } }
// Bootstrap ensures the dht routing table remains healthy as peers come and go. // it builds up a list of peers by requesting random peer IDs. The Bootstrap // process will run a number of queries each time, and run every time signal fires. // These parameters are configurable. // // As opposed to BootstrapWithConfig, Bootstrap satisfies the routing interface func (dht *IpfsDHT) Bootstrap(ctx context.Context) error { proc, err := dht.BootstrapWithConfig(DefaultBootstrapConfig) if err != nil { return err } // wait till ctx or dht.Context exits. // we have to do it this way to satisfy the Routing interface (contexts) go func() { defer proc.Close() select { case <-ctx.Done(): case <-dht.Context().Done(): } }() return nil }
func (rp *Reprovider) ProvideEvery(ctx context.Context, tick time.Duration) { // dont reprovide immediately. // may have just started the daemon and shutting it down immediately. // probability( up another minute | uptime ) increases with uptime. after := time.After(time.Minute) for { select { case <-ctx.Done(): return case <-after: err := rp.Reprovide(ctx) if err != nil { log.Debug(err) } after = time.After(tick) } } }
func (e *Engine) taskWorker(ctx context.Context) { defer close(e.outbox) // because taskWorker uses the channel exclusively for { oneTimeUse := make(chan *Envelope, 1) // buffer to prevent blocking select { case <-ctx.Done(): return case e.outbox <- oneTimeUse: } // receiver is ready for an outoing envelope. let's prepare one. first, // we must acquire a task from the PQ... envelope, err := e.nextEnvelope(ctx) if err != nil { close(oneTimeUse) return // ctx cancelled } oneTimeUse <- envelope // buffered. won't block close(oneTimeUse) } }
func (bs *Bitswap) provideWorker(ctx context.Context, id int) { idmap := eventlog.LoggableMap{"ID": id} for { log.Event(ctx, "Bitswap.ProvideWorker.Loop", idmap) select { case k, ok := <-bs.provideKeys: log.Event(ctx, "Bitswap.ProvideWorker.Work", idmap, &k) if !ok { log.Debug("provideKeys channel closed") return } ctx, cancel := context.WithTimeout(ctx, provideTimeout) err := bs.network.Provide(ctx, k) if err != nil { log.Error(err) } cancel() case <-ctx.Done(): return } } }
// GetBlock attempts to retrieve a particular block from peers within the // deadline enforced by the context. func (bs *Bitswap) GetBlock(parent context.Context, k key.Key) (*blocks.Block, error) { // Any async work initiated by this function must end when this function // returns. To ensure this, derive a new context. Note that it is okay to // listen on parent in this scope, but NOT okay to pass |parent| to // functions called by this one. Otherwise those functions won't return // when this context's cancel func is executed. This is difficult to // enforce. May this comment keep you safe. ctx, cancelFunc := context.WithCancel(parent) ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest")) log.Event(ctx, "Bitswap.GetBlockRequest.Start", &k) defer log.Event(ctx, "Bitswap.GetBlockRequest.End", &k) defer func() { cancelFunc() }() promise, err := bs.GetBlocks(ctx, []key.Key{k}) if err != nil { return nil, err } select { case block, ok := <-promise: if !ok { select { case <-ctx.Done(): return nil, ctx.Err() default: return nil, errors.New("promise channel was closed") } } return block, nil case <-parent.Done(): return nil, parent.Err() } }
// writeMsgCtx is used by the func writeMsgCtx(ctx context.Context, w msgio.Writer, msg proto.Message) ([]byte, error) { enc, err := proto.Marshal(msg) if err != nil { return nil, err } // write in a goroutine so we can exit when our context is cancelled. done := make(chan error) go func(m []byte) { err := w.WriteMsg(m) select { case done <- err: case <-ctx.Done(): } }(enc) select { case <-ctx.Done(): return nil, ctx.Err() case e := <-done: return enc, e } }
func (bs *Bitswap) taskWorker(ctx context.Context, id int) { idmap := eventlog.LoggableMap{"ID": id} defer log.Info("bitswap task worker shutting down...") for { log.Event(ctx, "Bitswap.TaskWorker.Loop", idmap) select { case nextEnvelope := <-bs.engine.Outbox(): select { case envelope, ok := <-nextEnvelope: if !ok { continue } log.Event(ctx, "Bitswap.TaskWorker.Work", eventlog.LoggableMap{"ID": id, "Target": envelope.Peer.Pretty(), "Block": envelope.Block.Multihash.B58String()}) bs.wm.SendBlock(ctx, envelope) case <-ctx.Done(): return } case <-ctx.Done(): return } } }
func (c *Client) FindProvidersAsync(ctx context.Context, k key.Key, max int) <-chan peer.PeerInfo { ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("findProviders")) defer log.EventBegin(ctx, "findProviders", &k).Done() ch := make(chan peer.PeerInfo) go func() { defer close(ch) request := pb.NewMessage(pb.Message_GET_PROVIDERS, string(k), 0) response, err := c.proxy.SendRequest(ctx, request) if err != nil { log.Debug(err) return } for _, p := range pb.PBPeersToPeerInfos(response.GetProviderPeers()) { select { case <-ctx.Done(): log.Debug(ctx.Err()) return case ch <- p: } } }() return ch }
func echoListen(ctx context.Context, listener Listener) { for { c, err := listener.Accept() if err != nil { select { case <-ctx.Done(): return default: } if ne, ok := err.(net.Error); ok && ne.Temporary() { <-time.After(time.Microsecond * 10) continue } log.Debugf("echoListen: listener appears to be closing") return } go echo(c.(Conn)) } }
// HasBlock announces the existance of a block to this bitswap service. The // service will potentially notify its peers. func (bs *Bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error { select { case <-bs.process.Closing(): return errors.New("bitswap is closed") default: } err := bs.tryPutBlock(blk, 4) // attempt to store block up to four times if err != nil { log.Errorf("Error writing block to datastore: %s", err) return err } bs.notifications.Publish(blk) select { case bs.newBlocks <- blk: // send block off to be reprovided case <-ctx.Done(): return ctx.Err() } return nil }
// nextEnvelope runs in the taskWorker goroutine. Returns an error if the // context is cancelled before the next Envelope can be created. func (e *Engine) nextEnvelope(ctx context.Context) (*Envelope, error) { for { nextTask := e.peerRequestQueue.Pop() for nextTask == nil { select { case <-ctx.Done(): return nil, ctx.Err() case <-e.workSignal: nextTask = e.peerRequestQueue.Pop() } } // with a task in hand, we're ready to prepare the envelope... block, err := e.bs.Get(nextTask.Entry.Key) if err != nil { // If we don't have the block, don't hold that against the peer // make sure to update that the task has been 'completed' nextTask.Done() continue } return &Envelope{ Peer: nextTask.Target, Block: block, Sent: func() { nextTask.Done() select { case e.workSignal <- struct{}{}: // work completing may mean that our queue will provide new // work to be done. default: } }, }, nil } }
func (dht *IpfsDHT) findProvidersAsyncRoutine(ctx context.Context, key key.Key, count int, peerOut chan peer.PeerInfo) { defer log.EventBegin(ctx, "findProvidersAsync", &key).Done() defer close(peerOut) ps := pset.NewLimited(count) provs := dht.providers.GetProviders(ctx, key) for _, p := range provs { // NOTE: assuming that this list of peers is unique if ps.TryAdd(p) { select { case peerOut <- dht.peerstore.PeerInfo(p): case <-ctx.Done(): return } } // If we have enough peers locally, dont bother with remote RPC if ps.Size() >= count { return } } // setup the Query query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { notif.PublishQueryEvent(ctx, ¬if.QueryEvent{ Type: notif.SendingQuery, ID: p, }) pmes, err := dht.findProvidersSingle(ctx, p, key) if err != nil { return nil, err } log.Debugf("%d provider entries", len(pmes.GetProviderPeers())) provs := pb.PBPeersToPeerInfos(pmes.GetProviderPeers()) log.Debugf("%d provider entries decoded", len(provs)) // Add unique providers from request, up to 'count' for _, prov := range provs { log.Debugf("got provider: %s", prov) if ps.TryAdd(prov.ID) { log.Debugf("using provider: %s", prov) select { case peerOut <- prov: case <-ctx.Done(): log.Debug("Context timed out sending more providers") return nil, ctx.Err() } } if ps.Size() >= count { log.Debugf("got enough providers (%d/%d)", ps.Size(), count) return &dhtQueryResult{success: true}, nil } } // Give closer peers back to the query to be queried closer := pmes.GetCloserPeers() clpeers := pb.PBPeersToPeerInfos(closer) log.Debugf("got closer peers: %d %s", len(clpeers), clpeers) notif.PublishQueryEvent(ctx, ¬if.QueryEvent{ Type: notif.PeerResponse, ID: p, Responses: pointerizePeerInfos(clpeers), }) return &dhtQueryResult{closerPeers: clpeers}, nil }) peers := dht.routingTable.NearestPeers(kb.ConvertKey(key), AlphaValue) _, err := query.Run(ctx, peers) if err != nil { log.Debugf("Query error: %s", err) notif.PublishQueryEvent(ctx, ¬if.QueryEvent{ Type: notif.QueryError, Extra: err.Error(), }) } }
// FindPeersConnectedToPeer searches for peers directly connected to a given peer. func (dht *IpfsDHT) FindPeersConnectedToPeer(ctx context.Context, id peer.ID) (<-chan peer.PeerInfo, error) { peerchan := make(chan peer.PeerInfo, asyncQueryBuffer) peersSeen := peer.Set{} peers := dht.routingTable.NearestPeers(kb.ConvertPeerID(id), AlphaValue) if len(peers) == 0 { return nil, kb.ErrLookupFailure } // setup the Query query := dht.newQuery(key.Key(id), func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { pmes, err := dht.findPeerSingle(ctx, p, id) if err != nil { return nil, err } var clpeers []peer.PeerInfo closer := pmes.GetCloserPeers() for _, pbp := range closer { pi := pb.PBPeerToPeerInfo(pbp) // skip peers already seen if _, found := peersSeen[pi.ID]; found { continue } peersSeen[pi.ID] = struct{}{} // if peer is connected, send it to our client. if pb.Connectedness(*pbp.Connection) == inet.Connected { select { case <-ctx.Done(): return nil, ctx.Err() case peerchan <- pi: } } // if peer is the peer we're looking for, don't bother querying it. // TODO maybe query it? if pb.Connectedness(*pbp.Connection) != inet.Connected { clpeers = append(clpeers, pi) } } return &dhtQueryResult{closerPeers: clpeers}, nil }) // run it! run it asynchronously to gen peers as results are found. // this does no error checking go func() { if _, err := query.Run(ctx, peers); err != nil { log.Debug(err) } // close the peerchan channel when done. close(peerchan) }() return peerchan, nil }