// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key key.Key) ([]byte, error) {
// If we have it local, dont bother doing an RPC!
val, err := dht.getLocal(key)
if err == nil {
log.Debug("have it locally")
return val, nil
} else {
log.Debug("failed to get value locally: %s", err)
}
// get closest peers in the routing table
rtp := dht.routingTable.NearestPeers(kb.ConvertKey(key), AlphaValue)
log.Debugf("peers in rt: %s", len(rtp), rtp)
if len(rtp) == 0 {
log.Warning("No peers from routing table!")
return nil, kb.ErrLookupFailure
}
// setup the Query
parent := ctx
query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) {
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.SendingQuery,
ID: p,
})
val, peers, err := dht.getValueOrPeers(ctx, p, key)
if err != nil {
return nil, err
}
res := &dhtQueryResult{value: val, closerPeers: peers}
if val != nil {
res.success = true
}
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.PeerResponse,
ID: p,
Responses: pointerizePeerInfos(peers),
})
return res, nil
})
// run it!
result, err := query.Run(ctx, rtp)
if err != nil {
return nil, err
}
log.Debugf("GetValue %v %v", key, result.value)
if result.value == nil {
return nil, routing.ErrNotFound
}
return result.value, nil
}
// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key key.Key, value []byte) error {
log.Debugf("PutValue %s", key)
sk, err := dht.getOwnPrivateKey()
if err != nil {
return err
}
sign, err := dht.Validator.IsSigned(key)
if err != nil {
return err
}
rec, err := record.MakePutRecord(sk, key, value, sign)
if err != nil {
log.Debug("Creation of record failed!")
return err
}
err = dht.putLocal(key, rec)
if err != nil {
return err
}
pchan, err := dht.GetClosestPeers(ctx, key)
if err != nil {
return err
}
wg := sync.WaitGroup{}
for p := range pchan {
wg.Add(1)
go func(p peer.ID) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
defer wg.Done()
notif.PublishQueryEvent(ctx, ¬if.QueryEvent{
Type: notif.Value,
ID: p,
})
err := dht.putValueToPeer(ctx, p, key, rec)
if err != nil {
log.Debugf("failed putting value to peer: %s", err)
}
}(p)
}
wg.Wait()
return nil
}
func (r *dhtQueryRunner) addPeerToQuery(next peer.ID) {
// if new peer is ourselves...
if next == r.query.dht.self {
r.log.Debug("addPeerToQuery skip self")
return
}
if !r.peersSeen.TryAdd(next) {
return
}
notif.PublishQueryEvent(r.runCtx, ¬if.QueryEvent{
Type: notif.AddingPeer,
ID: next,
})
r.peersRemaining.Increment(1)
select {
case r.peersToQuery.EnqChan <- next:
case <-r.proc.Closing():
}
}
func (r *dhtQueryRunner) queryPeer(proc process.Process, p peer.ID) {
// ok let's do this!
// create a context from our proc.
ctx := ctxproc.OnClosingContext(proc)
// make sure we do this when we exit
defer func() {
// signal we're done proccessing peer p
r.peersRemaining.Decrement(1)
r.rateLimit <- struct{}{}
}()
// make sure we're connected to the peer.
// FIXME abstract away into the network layer
if conns := r.query.dht.host.Network().ConnsToPeer(p); len(conns) == 0 {
log.Debug("not connected. dialing.")
notif.PublishQueryEvent(r.runCtx, ¬if.QueryEvent{
Type: notif.DialingPeer,
ID: p,
})
// while we dial, we do not take up a rate limit. this is to allow
// forward progress during potentially very high latency dials.
r.rateLimit <- struct{}{}
pi := peer.PeerInfo{ID: p}
if err := r.query.dht.host.Connect(ctx, pi); err != nil {
log.Debugf("Error connecting: %s", err)
notif.PublishQueryEvent(r.runCtx, ¬if.QueryEvent{
Type: notif.QueryError,
Extra: err.Error(),
ID: p,
})
r.Lock()
r.errs = append(r.errs, err)
r.Unlock()
<-r.rateLimit // need to grab it again, as we deferred.
return
}
<-r.rateLimit // need to grab it again, as we deferred.
log.Debugf("connected. dial success.")
}
// finally, run the query against this peer
res, err := r.query.qfunc(ctx, p)
if err != nil {
log.Debugf("ERROR worker for: %v %v", p, err)
r.Lock()
r.errs = append(r.errs, err)
r.Unlock()
} else if res.success {
log.Debugf("SUCCESS worker for: %v %s", p, res)
r.Lock()
r.result = res
r.Unlock()
go r.proc.Close() // signal to everyone that we're done.
// must be async, as we're one of the children, and Close blocks.
} else if len(res.closerPeers) > 0 {
log.Debugf("PEERS CLOSER -- worker for: %v (%d closer peers)", p, len(res.closerPeers))
for _, next := range res.closerPeers {
if next.ID == r.query.dht.self { // dont add self.
log.Debugf("PEERS CLOSER -- worker for: %v found self", p)
continue
}
// add their addresses to the dialer's peerstore
r.query.dht.peerstore.AddAddrs(next.ID, next.Addrs, peer.TempAddrTTL)
r.addPeerToQuery(next.ID)
log.Debugf("PEERS CLOSER -- worker for: %v added %v (%v)", p, next.ID, next.Addrs)
}
} else {
log.Debugf("QUERY worker for: %v - not found, and no closer peers.", p)
}
}
// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.PeerInfo, error) {
defer log.EventBegin(ctx, "FindPeer", id).Done()
// Check if were already connected to them
if pi := dht.FindLocal(id); pi.ID != "" {
return pi, nil
}
peers := dht.routingTable.NearestPeers(kb.ConvertPeerID(id), AlphaValue)
if len(peers) == 0 {
return peer.PeerInfo{}, kb.ErrLookupFailure
}
// Sanity...
for _, p := range peers {
if p == id {
log.Debug("Found target peer in list of closest peers...")
return dht.peerstore.PeerInfo(p), nil
}
}
// setup the Query
parent := ctx
query := dht.newQuery(key.Key(id), func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) {
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.SendingQuery,
ID: p,
})
pmes, err := dht.findPeerSingle(ctx, p, id)
if err != nil {
return nil, err
}
closer := pmes.GetCloserPeers()
clpeerInfos := pb.PBPeersToPeerInfos(closer)
// see it we got the peer here
for _, npi := range clpeerInfos {
if npi.ID == id {
return &dhtQueryResult{
peer: npi,
success: true,
}, nil
}
}
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.PeerResponse,
Responses: pointerizePeerInfos(clpeerInfos),
})
return &dhtQueryResult{closerPeers: clpeerInfos}, nil
})
// run it!
result, err := query.Run(ctx, peers)
if err != nil {
return peer.PeerInfo{}, err
}
log.Debugf("FindPeer %v %v", id, result.success)
if result.peer.ID == "" {
return peer.PeerInfo{}, routing.ErrNotFound
}
return result.peer, nil
}
func (dht *IpfsDHT) findProvidersAsyncRoutine(ctx context.Context, key key.Key, count int, peerOut chan peer.PeerInfo) {
defer log.EventBegin(ctx, "findProvidersAsync", &key).Done()
defer close(peerOut)
ps := pset.NewLimited(count)
provs := dht.providers.GetProviders(ctx, key)
for _, p := range provs {
// NOTE: assuming that this list of peers is unique
if ps.TryAdd(p) {
select {
case peerOut <- dht.peerstore.PeerInfo(p):
case <-ctx.Done():
return
}
}
// If we have enough peers locally, dont bother with remote RPC
if ps.Size() >= count {
return
}
}
// setup the Query
parent := ctx
query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) {
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.SendingQuery,
ID: p,
})
pmes, err := dht.findProvidersSingle(ctx, p, key)
if err != nil {
return nil, err
}
log.Debugf("%d provider entries", len(pmes.GetProviderPeers()))
provs := pb.PBPeersToPeerInfos(pmes.GetProviderPeers())
log.Debugf("%d provider entries decoded", len(provs))
// Add unique providers from request, up to 'count'
for _, prov := range provs {
log.Debugf("got provider: %s", prov)
if ps.TryAdd(prov.ID) {
log.Debugf("using provider: %s", prov)
select {
case peerOut <- prov:
case <-ctx.Done():
log.Debug("Context timed out sending more providers")
return nil, ctx.Err()
}
}
if ps.Size() >= count {
log.Debugf("got enough providers (%d/%d)", ps.Size(), count)
return &dhtQueryResult{success: true}, nil
}
}
// Give closer peers back to the query to be queried
closer := pmes.GetCloserPeers()
clpeers := pb.PBPeersToPeerInfos(closer)
log.Debugf("got closer peers: %d %s", len(clpeers), clpeers)
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.PeerResponse,
ID: p,
Responses: pointerizePeerInfos(clpeers),
})
return &dhtQueryResult{closerPeers: clpeers}, nil
})
peers := dht.routingTable.NearestPeers(kb.ConvertKey(key), AlphaValue)
_, err := query.Run(ctx, peers)
if err != nil {
log.Debugf("Query error: %s", err)
notif.PublishQueryEvent(ctx, ¬if.QueryEvent{
Type: notif.QueryError,
Extra: err.Error(),
})
}
}
func (dht *IpfsDHT) GetValues(ctx context.Context, key key.Key, nvals int) ([]routing.RecvdVal, error) {
var vals []routing.RecvdVal
var valslock sync.Mutex
// If we have it local, dont bother doing an RPC!
lrec, err := dht.getLocal(key)
if err == nil {
// TODO: this is tricky, we dont always want to trust our own value
// what if the authoritative source updated it?
log.Debug("have it locally")
vals = append(vals, routing.RecvdVal{
Val: lrec.GetValue(),
From: dht.self,
})
if nvals <= 1 {
return vals, nil
}
} else if nvals == 0 {
return nil, err
}
// get closest peers in the routing table
rtp := dht.routingTable.NearestPeers(kb.ConvertKey(key), KValue)
log.Debugf("peers in rt: %s", len(rtp), rtp)
if len(rtp) == 0 {
log.Warning("No peers from routing table!")
return nil, kb.ErrLookupFailure
}
// setup the Query
parent := ctx
query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) {
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.SendingQuery,
ID: p,
})
rec, peers, err := dht.getValueOrPeers(ctx, p, key)
switch err {
case routing.ErrNotFound:
// in this case, they responded with nothing,
// still send a notification so listeners can know the
// request has completed 'successfully'
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.PeerResponse,
ID: p,
})
return nil, err
default:
return nil, err
case nil, errInvalidRecord:
// in either of these cases, we want to keep going
}
res := &dhtQueryResult{closerPeers: peers}
if rec.GetValue() != nil || err == errInvalidRecord {
rv := routing.RecvdVal{
Val: rec.GetValue(),
From: p,
}
valslock.Lock()
vals = append(vals, rv)
// If weve collected enough records, we're done
if len(vals) >= nvals {
res.success = true
}
valslock.Unlock()
}
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.PeerResponse,
ID: p,
Responses: pointerizePeerInfos(peers),
})
return res, nil
})
// run it!
_, err = query.Run(ctx, rtp)
if len(vals) == 0 {
if err != nil {
return nil, err
}
}
return vals, nil
}
// Kademlia 'node lookup' operation. Returns a channel of the K closest peers
// to the given key
func (dht *IpfsDHT) GetClosestPeers(ctx context.Context, key key.Key) (<-chan peer.ID, error) {
e := log.EventBegin(ctx, "getClosestPeers", &key)
tablepeers := dht.routingTable.NearestPeers(kb.ConvertKey(key), KValue)
if len(tablepeers) == 0 {
return nil, kb.ErrLookupFailure
}
out := make(chan peer.ID, KValue)
peerset := pset.NewLimited(KValue)
for _, p := range tablepeers {
select {
case out <- p:
case <-ctx.Done():
return nil, ctx.Err()
}
peerset.Add(p)
}
// since the query doesnt actually pass our context down
// we have to hack this here. whyrusleeping isnt a huge fan of goprocess
parent := ctx
query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) {
// For DHT query command
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.SendingQuery,
ID: p,
})
closer, err := dht.closerPeersSingle(ctx, key, p)
if err != nil {
log.Debugf("error getting closer peers: %s", err)
return nil, err
}
var filtered []peer.PeerInfo
for _, clp := range closer {
if kb.Closer(clp, dht.self, key) && peerset.TryAdd(clp) {
select {
case out <- clp:
case <-ctx.Done():
return nil, ctx.Err()
}
filtered = append(filtered, dht.peerstore.PeerInfo(clp))
}
}
// For DHT query command
notif.PublishQueryEvent(parent, ¬if.QueryEvent{
Type: notif.PeerResponse,
ID: p,
Responses: pointerizePeerInfos(filtered),
})
return &dhtQueryResult{closerPeers: filtered}, nil
})
go func() {
defer close(out)
defer e.Done()
// run it!
_, err := query.Run(ctx, tablepeers)
if err != nil {
log.Debugf("closestPeers query run error: %s", err)
}
}()
return out, nil
}
}
events := make(chan *notif.QueryEvent)
ctx := notif.RegisterForQueryEvents(req.Context(), events)
closestPeers, err := dht.GetClosestPeers(ctx, key.Key(req.Arguments()[0]))
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
go func() {
defer close(events)
for p := range closestPeers {
notif.PublishQueryEvent(ctx, ¬if.QueryEvent{
ID: p,
Type: notif.FinalPeer,
})
}
}()
outChan := make(chan interface{})
res.SetOutput((<-chan interface{})(outChan))
go func() {
defer close(outChan)
for e := range events {
outChan <- e
}
}()
},
Marshalers: cmds.MarshalerMap{