// GetValue searches for the value corresponding to given Key. // If the search does not succeed, a multiaddr string of a closer peer is // returned along with util.ErrSearchIncomplete func (dht *IpfsDHT) GetValue(ctx context.Context, key key.Key) ([]byte, error) { // If we have it local, dont bother doing an RPC! val, err := dht.getLocal(key) if err == nil { log.Debug("have it locally") return val, nil } else { log.Debug("failed to get value locally: %s", err) } // get closest peers in the routing table rtp := dht.routingTable.NearestPeers(kb.ConvertKey(key), AlphaValue) log.Debugf("peers in rt: %s", len(rtp), rtp) if len(rtp) == 0 { log.Warning("No peers from routing table!") return nil, kb.ErrLookupFailure } // setup the Query parent := ctx query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.SendingQuery, ID: p, }) val, peers, err := dht.getValueOrPeers(ctx, p, key) if err != nil { return nil, err } res := &dhtQueryResult{value: val, closerPeers: peers} if val != nil { res.success = true } notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.PeerResponse, ID: p, Responses: pointerizePeerInfos(peers), }) return res, nil }) // run it! result, err := query.Run(ctx, rtp) if err != nil { return nil, err } log.Debugf("GetValue %v %v", key, result.value) if result.value == nil { return nil, routing.ErrNotFound } return result.value, nil }
// PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT func (dht *IpfsDHT) PutValue(ctx context.Context, key key.Key, value []byte) error { log.Debugf("PutValue %s", key) sk, err := dht.getOwnPrivateKey() if err != nil { return err } sign, err := dht.Validator.IsSigned(key) if err != nil { return err } rec, err := record.MakePutRecord(sk, key, value, sign) if err != nil { log.Debug("Creation of record failed!") return err } err = dht.putLocal(key, rec) if err != nil { return err } pchan, err := dht.GetClosestPeers(ctx, key) if err != nil { return err } wg := sync.WaitGroup{} for p := range pchan { wg.Add(1) go func(p peer.ID) { ctx, cancel := context.WithCancel(ctx) defer cancel() defer wg.Done() notif.PublishQueryEvent(ctx, ¬if.QueryEvent{ Type: notif.Value, ID: p, }) err := dht.putValueToPeer(ctx, p, key, rec) if err != nil { log.Debugf("failed putting value to peer: %s", err) } }(p) } wg.Wait() return nil }
func (r *dhtQueryRunner) addPeerToQuery(next peer.ID) { // if new peer is ourselves... if next == r.query.dht.self { r.log.Debug("addPeerToQuery skip self") return } if !r.peersSeen.TryAdd(next) { return } notif.PublishQueryEvent(r.runCtx, ¬if.QueryEvent{ Type: notif.AddingPeer, ID: next, }) r.peersRemaining.Increment(1) select { case r.peersToQuery.EnqChan <- next: case <-r.proc.Closing(): } }
func (r *dhtQueryRunner) queryPeer(proc process.Process, p peer.ID) { // ok let's do this! // create a context from our proc. ctx := ctxproc.OnClosingContext(proc) // make sure we do this when we exit defer func() { // signal we're done proccessing peer p r.peersRemaining.Decrement(1) r.rateLimit <- struct{}{} }() // make sure we're connected to the peer. // FIXME abstract away into the network layer if conns := r.query.dht.host.Network().ConnsToPeer(p); len(conns) == 0 { log.Debug("not connected. dialing.") notif.PublishQueryEvent(r.runCtx, ¬if.QueryEvent{ Type: notif.DialingPeer, ID: p, }) // while we dial, we do not take up a rate limit. this is to allow // forward progress during potentially very high latency dials. r.rateLimit <- struct{}{} pi := peer.PeerInfo{ID: p} if err := r.query.dht.host.Connect(ctx, pi); err != nil { log.Debugf("Error connecting: %s", err) notif.PublishQueryEvent(r.runCtx, ¬if.QueryEvent{ Type: notif.QueryError, Extra: err.Error(), ID: p, }) r.Lock() r.errs = append(r.errs, err) r.Unlock() <-r.rateLimit // need to grab it again, as we deferred. return } <-r.rateLimit // need to grab it again, as we deferred. log.Debugf("connected. dial success.") } // finally, run the query against this peer res, err := r.query.qfunc(ctx, p) if err != nil { log.Debugf("ERROR worker for: %v %v", p, err) r.Lock() r.errs = append(r.errs, err) r.Unlock() } else if res.success { log.Debugf("SUCCESS worker for: %v %s", p, res) r.Lock() r.result = res r.Unlock() go r.proc.Close() // signal to everyone that we're done. // must be async, as we're one of the children, and Close blocks. } else if len(res.closerPeers) > 0 { log.Debugf("PEERS CLOSER -- worker for: %v (%d closer peers)", p, len(res.closerPeers)) for _, next := range res.closerPeers { if next.ID == r.query.dht.self { // dont add self. log.Debugf("PEERS CLOSER -- worker for: %v found self", p) continue } // add their addresses to the dialer's peerstore r.query.dht.peerstore.AddAddrs(next.ID, next.Addrs, peer.TempAddrTTL) r.addPeerToQuery(next.ID) log.Debugf("PEERS CLOSER -- worker for: %v added %v (%v)", p, next.ID, next.Addrs) } } else { log.Debugf("QUERY worker for: %v - not found, and no closer peers.", p) } }
// FindPeer searches for a peer with given ID. func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.PeerInfo, error) { defer log.EventBegin(ctx, "FindPeer", id).Done() // Check if were already connected to them if pi := dht.FindLocal(id); pi.ID != "" { return pi, nil } peers := dht.routingTable.NearestPeers(kb.ConvertPeerID(id), AlphaValue) if len(peers) == 0 { return peer.PeerInfo{}, kb.ErrLookupFailure } // Sanity... for _, p := range peers { if p == id { log.Debug("Found target peer in list of closest peers...") return dht.peerstore.PeerInfo(p), nil } } // setup the Query parent := ctx query := dht.newQuery(key.Key(id), func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.SendingQuery, ID: p, }) pmes, err := dht.findPeerSingle(ctx, p, id) if err != nil { return nil, err } closer := pmes.GetCloserPeers() clpeerInfos := pb.PBPeersToPeerInfos(closer) // see it we got the peer here for _, npi := range clpeerInfos { if npi.ID == id { return &dhtQueryResult{ peer: npi, success: true, }, nil } } notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.PeerResponse, Responses: pointerizePeerInfos(clpeerInfos), }) return &dhtQueryResult{closerPeers: clpeerInfos}, nil }) // run it! result, err := query.Run(ctx, peers) if err != nil { return peer.PeerInfo{}, err } log.Debugf("FindPeer %v %v", id, result.success) if result.peer.ID == "" { return peer.PeerInfo{}, routing.ErrNotFound } return result.peer, nil }
func (dht *IpfsDHT) findProvidersAsyncRoutine(ctx context.Context, key key.Key, count int, peerOut chan peer.PeerInfo) { defer log.EventBegin(ctx, "findProvidersAsync", &key).Done() defer close(peerOut) ps := pset.NewLimited(count) provs := dht.providers.GetProviders(ctx, key) for _, p := range provs { // NOTE: assuming that this list of peers is unique if ps.TryAdd(p) { select { case peerOut <- dht.peerstore.PeerInfo(p): case <-ctx.Done(): return } } // If we have enough peers locally, dont bother with remote RPC if ps.Size() >= count { return } } // setup the Query parent := ctx query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.SendingQuery, ID: p, }) pmes, err := dht.findProvidersSingle(ctx, p, key) if err != nil { return nil, err } log.Debugf("%d provider entries", len(pmes.GetProviderPeers())) provs := pb.PBPeersToPeerInfos(pmes.GetProviderPeers()) log.Debugf("%d provider entries decoded", len(provs)) // Add unique providers from request, up to 'count' for _, prov := range provs { log.Debugf("got provider: %s", prov) if ps.TryAdd(prov.ID) { log.Debugf("using provider: %s", prov) select { case peerOut <- prov: case <-ctx.Done(): log.Debug("Context timed out sending more providers") return nil, ctx.Err() } } if ps.Size() >= count { log.Debugf("got enough providers (%d/%d)", ps.Size(), count) return &dhtQueryResult{success: true}, nil } } // Give closer peers back to the query to be queried closer := pmes.GetCloserPeers() clpeers := pb.PBPeersToPeerInfos(closer) log.Debugf("got closer peers: %d %s", len(clpeers), clpeers) notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.PeerResponse, ID: p, Responses: pointerizePeerInfos(clpeers), }) return &dhtQueryResult{closerPeers: clpeers}, nil }) peers := dht.routingTable.NearestPeers(kb.ConvertKey(key), AlphaValue) _, err := query.Run(ctx, peers) if err != nil { log.Debugf("Query error: %s", err) notif.PublishQueryEvent(ctx, ¬if.QueryEvent{ Type: notif.QueryError, Extra: err.Error(), }) } }
func (dht *IpfsDHT) GetValues(ctx context.Context, key key.Key, nvals int) ([]routing.RecvdVal, error) { var vals []routing.RecvdVal var valslock sync.Mutex // If we have it local, dont bother doing an RPC! lrec, err := dht.getLocal(key) if err == nil { // TODO: this is tricky, we dont always want to trust our own value // what if the authoritative source updated it? log.Debug("have it locally") vals = append(vals, routing.RecvdVal{ Val: lrec.GetValue(), From: dht.self, }) if nvals <= 1 { return vals, nil } } else if nvals == 0 { return nil, err } // get closest peers in the routing table rtp := dht.routingTable.NearestPeers(kb.ConvertKey(key), KValue) log.Debugf("peers in rt: %s", len(rtp), rtp) if len(rtp) == 0 { log.Warning("No peers from routing table!") return nil, kb.ErrLookupFailure } // setup the Query parent := ctx query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.SendingQuery, ID: p, }) rec, peers, err := dht.getValueOrPeers(ctx, p, key) switch err { case routing.ErrNotFound: // in this case, they responded with nothing, // still send a notification so listeners can know the // request has completed 'successfully' notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.PeerResponse, ID: p, }) return nil, err default: return nil, err case nil, errInvalidRecord: // in either of these cases, we want to keep going } res := &dhtQueryResult{closerPeers: peers} if rec.GetValue() != nil || err == errInvalidRecord { rv := routing.RecvdVal{ Val: rec.GetValue(), From: p, } valslock.Lock() vals = append(vals, rv) // If weve collected enough records, we're done if len(vals) >= nvals { res.success = true } valslock.Unlock() } notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.PeerResponse, ID: p, Responses: pointerizePeerInfos(peers), }) return res, nil }) // run it! _, err = query.Run(ctx, rtp) if len(vals) == 0 { if err != nil { return nil, err } } return vals, nil }
// Kademlia 'node lookup' operation. Returns a channel of the K closest peers // to the given key func (dht *IpfsDHT) GetClosestPeers(ctx context.Context, key key.Key) (<-chan peer.ID, error) { e := log.EventBegin(ctx, "getClosestPeers", &key) tablepeers := dht.routingTable.NearestPeers(kb.ConvertKey(key), KValue) if len(tablepeers) == 0 { return nil, kb.ErrLookupFailure } out := make(chan peer.ID, KValue) peerset := pset.NewLimited(KValue) for _, p := range tablepeers { select { case out <- p: case <-ctx.Done(): return nil, ctx.Err() } peerset.Add(p) } // since the query doesnt actually pass our context down // we have to hack this here. whyrusleeping isnt a huge fan of goprocess parent := ctx query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { // For DHT query command notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.SendingQuery, ID: p, }) closer, err := dht.closerPeersSingle(ctx, key, p) if err != nil { log.Debugf("error getting closer peers: %s", err) return nil, err } var filtered []peer.PeerInfo for _, clp := range closer { if kb.Closer(clp, dht.self, key) && peerset.TryAdd(clp) { select { case out <- clp: case <-ctx.Done(): return nil, ctx.Err() } filtered = append(filtered, dht.peerstore.PeerInfo(clp)) } } // For DHT query command notif.PublishQueryEvent(parent, ¬if.QueryEvent{ Type: notif.PeerResponse, ID: p, Responses: pointerizePeerInfos(filtered), }) return &dhtQueryResult{closerPeers: filtered}, nil }) go func() { defer close(out) defer e.Done() // run it! _, err := query.Run(ctx, tablepeers) if err != nil { log.Debugf("closestPeers query run error: %s", err) } }() return out, nil }
} events := make(chan *notif.QueryEvent) ctx := notif.RegisterForQueryEvents(req.Context(), events) closestPeers, err := dht.GetClosestPeers(ctx, key.Key(req.Arguments()[0])) if err != nil { res.SetError(err, cmds.ErrNormal) return } go func() { defer close(events) for p := range closestPeers { notif.PublishQueryEvent(ctx, ¬if.QueryEvent{ ID: p, Type: notif.FinalPeer, }) } }() outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) go func() { defer close(outChan) for e := range events { outChan <- e } }() }, Marshalers: cmds.MarshalerMap{