// sendRequest sends out a request using dht.sender, but also makes sure to // measure the RTT for latency measurements. func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.Peer, pmes *pb.Message) (*pb.Message, error) { mes, err := msg.FromObject(p, pmes) if err != nil { return nil, err } start := time.Now() log.Event(ctx, "sentMessage", dht.self, p, pmes) rmes, err := dht.sender.SendRequest(ctx, mes) if err != nil { return nil, err } if rmes == nil { return nil, errors.New("no response to request") } rtt := time.Since(start) rmes.Peer().SetLatency(rtt) rpmes := new(pb.Message) if err := proto.Unmarshal(rmes.Data(), rpmes); err != nil { return nil, err } return rpmes, nil }
func TestMultiBlock(t *testing.T) { mbf := new(MultiBlock) for i := 0; i < 15; i++ { mbf.AddBlockSize(100) } mbf.Data = make([]byte, 128) b, err := mbf.GetBytes() if err != nil { t.Fatal(err) } pbn := new(pb.Data) err = proto.Unmarshal(b, pbn) if err != nil { t.Fatal(err) } ds, err := DataSize(b) if err != nil { t.Fatal(err) } if ds != (100*15)+128 { t.Fatal("Datasize calculations incorrect!") } }
// getLocal attempts to retrieve the value from the datastore func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) { dht.dslock.Lock() defer dht.dslock.Unlock() log.Debug("getLocal %s", key) v, err := dht.datastore.Get(key.DsKey()) if err != nil { return nil, err } log.Debug("found in db") byt, ok := v.([]byte) if !ok { return nil, errors.New("value stored in datastore not []byte") } rec := new(pb.Record) err = proto.Unmarshal(byt, rec) if err != nil { return nil, err } // TODO: 'if paranoid' if u.Debug { err = dht.verifyRecord(rec) if err != nil { log.Errorf("local record verify failed: %s", err) return nil, err } } return rec.GetValue(), nil }
// precalcNextBuf follows the next link in line and loads it from the DAGService, // setting the next buffer to read from func (dr *DagReader) precalcNextBuf() error { if dr.position >= len(dr.node.Links) { return io.EOF } nxt, err := dr.node.Links[dr.position].GetNode(dr.serv) if err != nil { return err } pb := new(ftpb.Data) err = proto.Unmarshal(nxt.Data, pb) if err != nil { return err } dr.position++ switch pb.GetType() { case ftpb.Data_Directory: // A directory should not exist within a file return ft.ErrInvalidDirLocation case ftpb.Data_File: //TODO: this *should* work, needs testing first log.Warning("Running untested code for multilayered indirect FS reads.") subr, err := NewDagReader(nxt, dr.serv) if err != nil { return err } dr.buf = subr return nil case ftpb.Data_Raw: dr.buf = bytes.NewBuffer(pb.GetData()) return nil default: return ft.ErrUnrecognizedType } }
func UnwrapData(data []byte) ([]byte, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) if err != nil { return nil, err } return pbdata.GetData(), nil }
func FromNet(nmsg netmsg.NetMessage) (BitSwapMessage, error) { pb := new(pb.Message) if err := proto.Unmarshal(nmsg.Data(), pb); err != nil { return nil, err } m := newMessageFromProto(*pb) return m, nil }
func FromBytes(data []byte) (*pb.Data, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) if err != nil { return nil, err } return pbdata, nil }
// HandleMessage implements the inet.Handler interface. func (dht *IpfsDHT) HandleMessage(ctx context.Context, mes msg.NetMessage) msg.NetMessage { mData := mes.Data() if mData == nil { log.Error("Message contained nil data.") return nil } mPeer := mes.Peer() if mPeer == nil { log.Error("Message contained nil peer.") return nil } // deserialize msg pmes := new(pb.Message) err := proto.Unmarshal(mData, pmes) if err != nil { log.Error("Error unmarshaling data") return nil } // update the peer (on valid msgs only) dht.Update(ctx, mPeer) log.Event(ctx, "foo", dht.self, mPeer, pmes) // get handler for this msg type. handler := dht.handlerForMsgType(pmes.GetType()) if handler == nil { log.Error("got back nil handler from handlerForMsgType") return nil } // dispatch handler. rpmes, err := handler(mPeer, pmes) if err != nil { log.Errorf("handle message error: %s", err) return nil } // if nil response, return it before serializing if rpmes == nil { log.Warning("Got back nil response from request.") return nil } // serialize response msg rmes, err := msg.FromObject(mPeer, rpmes) if err != nil { log.Errorf("serialze response error: %s", err) return nil } return rmes }
func unwrapData(data []byte) ([]byte, pb.ProtocolID, error) { // Unmarshal pbm := new(pb.PBProtocolMessage) err := proto.Unmarshal(data, pbm) if err != nil { return nil, 0, err } return pbm.GetData(), pbm.GetProtocolID(), nil }
func unwrapData(data []byte) ([]byte, RequestID, error) { // Unmarshal pbm := new(pb.PBRequest) err := proto.Unmarshal(data, pbm) if err != nil { return nil, nil, err } return pbm.GetData(), pbm.GetTag(), nil }
// Resolve implements Resolver. Uses the IPFS routing system to resolve SFS-like // names. func (r *routingResolver) Resolve(name string) (string, error) { log.Debugf("RoutingResolve: '%s'", name) ctx := context.TODO() hash, err := mh.FromB58String(name) if err != nil { log.Warning("RoutingResolve: bad input hash: [%s]\n", name) return "", err } // name should be a multihash. if it isn't, error out here. // use the routing system to get the name. // /ipns/<name> h := []byte("/ipns/" + string(hash)) ipnsKey := u.Key(h) val, err := r.routing.GetValue(ctx, ipnsKey) if err != nil { log.Warning("RoutingResolve get failed.") return "", err } entry := new(pb.IpnsEntry) err = proto.Unmarshal(val, entry) if err != nil { return "", err } // name should be a public key retrievable from ipfs // /ipfs/<name> key := u.Key("/pk/" + string(hash)) pkval, err := r.routing.GetValue(ctx, key) if err != nil { log.Warning("RoutingResolve PubKey Get failed.") return "", err } // get PublicKey from node.Data pk, err := ci.UnmarshalPublicKey(pkval) if err != nil { return "", err } hsh, _ := pk.Hash() log.Debugf("pk hash = %s", u.Key(hsh)) // check sig with pk if ok, err := pk.Verify(ipnsEntryDataForSig(entry), entry.GetSignature()); err != nil || !ok { return "", fmt.Errorf("Invalid value. Not signed by PrivateKey corresponding to %v", pk) } // ok sig checks out. this is a valid name. return string(entry.GetValue()), nil }
// UnmarshalPrivateKey converts a protobuf serialized private key into its // representative object func UnmarshalPrivateKey(data []byte) (PrivKey, error) { pmes := new(pb.PrivateKey) err := proto.Unmarshal(data, pmes) if err != nil { return nil, err } switch pmes.GetType() { case pb.KeyType_RSA: return UnmarshalRsaPrivateKey(pmes.GetData()) default: return nil, ErrBadKeyType } }
// Handshake3 exchanges local and remote service information func Handshake3(ctx context.Context, c Conn) (*handshake.Handshake3Result, error) { rpeer := c.RemotePeer() lpeer := c.LocalPeer() // setup + send the message to remote var remoteH, localH *hspb.Handshake3 localH = handshake.Handshake3Msg(lpeer, c.RemoteMultiaddr()) localB, err := proto.Marshal(localH) if err != nil { return nil, err } c.Out() <- localB log.Debugf("Handshake1: sent to %s", rpeer) // wait + listen for response select { case <-ctx.Done(): return nil, ctx.Err() case <-c.Closing(): return nil, errors.New("Handshake3: error remote connection closed") case remoteB, ok := <-c.In(): if !ok { return nil, fmt.Errorf("Handshake3 error receiving from conn: %v", rpeer) } remoteH = new(hspb.Handshake3) err = proto.Unmarshal(remoteB, remoteH) if err != nil { return nil, fmt.Errorf("Handshake3 could not decode remote msg: %q", err) } log.Debugf("Handshake3 received from %s", rpeer) } // actually update our state based on the new knowledge res, err := handshake.Handshake3Update(lpeer, rpeer, remoteH) if err != nil { log.Errorf("Handshake3 failed to update %s", rpeer) } res.RemoteObservedAddress = c.RemoteMultiaddr() return res, nil }
func DataSize(data []byte) (uint64, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) if err != nil { return 0, err } switch pbdata.GetType() { case pb.Data_Directory: return 0, errors.New("Cant get data size of directory!") case pb.Data_File: return pbdata.GetFilesize(), nil case pb.Data_Raw: return uint64(len(pbdata.GetData())), nil default: return 0, errors.New("Unrecognized node data type!") } }
// Handshake1 exchanges local and remote versions and compares them // closes remote and returns an error in case of major difference func Handshake1(ctx context.Context, c Conn) error { rpeer := c.RemotePeer() lpeer := c.LocalPeer() var remoteH, localH *hspb.Handshake1 localH = handshake.Handshake1Msg() myVerBytes, err := proto.Marshal(localH) if err != nil { return err } c.Out() <- myVerBytes log.Debugf("Sent my version (%s) to %s", localH, rpeer) select { case <-ctx.Done(): return ctx.Err() case <-c.Closing(): return errors.New("remote closed connection during version exchange") case data, ok := <-c.In(): if !ok { return fmt.Errorf("error retrieving from conn: %v", rpeer) } remoteH = new(hspb.Handshake1) err = proto.Unmarshal(data, remoteH) if err != nil { return fmt.Errorf("could not decode remote version: %q", err) } log.Debugf("Received remote version (%s) from %s", remoteH, rpeer) } if err := handshake.Handshake1Compatible(localH, remoteH); err != nil { log.Infof("%s (%s) incompatible version with %s (%s)", lpeer, localH, rpeer, remoteH) return err } log.Debugf("%s version handshake compatible %s", lpeer, rpeer) return nil }
// ValidateIpnsRecord implements ValidatorFunc and verifies that the // given 'val' is an IpnsEntry and that that entry is valid. func ValidateIpnsRecord(k u.Key, val []byte) error { entry := new(pb.IpnsEntry) err := proto.Unmarshal(val, entry) if err != nil { return err } switch entry.GetValidityType() { case pb.IpnsEntry_EOL: t, err := u.ParseRFC3339(string(entry.GetValue())) if err != nil { log.Error("Failed parsing time for ipns record EOL") return err } if time.Now().After(t) { return ErrExpiredRecord } default: return ErrUnrecognizedValidity } return nil }
// NewDagReader creates a new reader object that reads the data represented by the given // node, using the passed in DAGService for data retreival func NewDagReader(n *mdag.Node, serv mdag.DAGService) (io.Reader, error) { pb := new(ftpb.Data) err := proto.Unmarshal(n.Data, pb) if err != nil { return nil, err } switch pb.GetType() { case ftpb.Data_Directory: // Dont allow reading directories return nil, ErrIsDir case ftpb.Data_File: return &DagReader{ node: n, serv: serv, buf: bytes.NewBuffer(pb.GetData()), }, nil case ftpb.Data_Raw: // Raw block will just be a single level, return a byte buffer return bytes.NewBuffer(pb.GetData()), nil default: return nil, ft.ErrUnrecognizedType } }
// handsahke performs initial communication over insecure channel to share // keys, IDs, and initiate communication. func (s *SecurePipe) handshake() error { // Generate and send Hello packet. // Hello = (rand, PublicKey, Supported) nonce := make([]byte, 16) _, err := rand.Read(nonce) if err != nil { return err } log.Debugf("handshake: %s <--> %s", s.local, s.remote) myPubKey, err := s.local.PubKey().Bytes() if err != nil { return err } proposeMsg := new(pb.Propose) proposeMsg.Rand = nonce proposeMsg.Pubkey = myPubKey proposeMsg.Exchanges = &SupportedExchanges proposeMsg.Ciphers = &SupportedCiphers proposeMsg.Hashes = &SupportedHashes encoded, err := proto.Marshal(proposeMsg) if err != nil { return err } // Send our Propose packet select { case s.insecure.Out <- encoded: case <-s.ctx.Done(): return ErrClosed } // Parse their Propose packet and generate an Exchange packet. // Exchange = (EphemeralPubKey, Signature) var resp []byte select { case <-s.ctx.Done(): return ErrClosed case resp = <-s.insecure.In: } // u.POut("received encoded handshake\n") proposeResp := new(pb.Propose) err = proto.Unmarshal(resp, proposeResp) if err != nil { return err } // get remote identity remotePubKey, err := ci.UnmarshalPublicKey(proposeResp.GetPubkey()) if err != nil { return err } // get or construct peer s.remote, err = getOrConstructPeer(s.peers, remotePubKey) if err != nil { return err } log.Debugf("%s Remote Peer Identified as %s", s.local, s.remote) exchange, err := SelectBest(SupportedExchanges, proposeResp.GetExchanges()) if err != nil { return err } cipherType, err := SelectBest(SupportedCiphers, proposeResp.GetCiphers()) if err != nil { return err } hashType, err := SelectBest(SupportedHashes, proposeResp.GetHashes()) if err != nil { return err } // u.POut("Selected %s %s %s\n", exchange, cipherType, hashType) epubkey, genSharedKey, err := ci.GenerateEKeyPair(exchange) // Generate EphemeralPubKey var handshake bytes.Buffer // Gather corpus to sign. handshake.Write(encoded) handshake.Write(resp) handshake.Write(epubkey) exPacket := new(pb.Exchange) exPacket.Epubkey = epubkey exPacket.Signature, err = s.local.PrivKey().Sign(handshake.Bytes()) if err != nil { return err } exEncoded, err := proto.Marshal(exPacket) // send out Exchange packet select { case s.insecure.Out <- exEncoded: case <-s.ctx.Done(): return ErrClosed } // Parse their Exchange packet and generate a Finish packet. // Finish = E('Finish') var resp1 []byte select { case <-s.ctx.Done(): return ErrClosed case resp1 = <-s.insecure.In: } exchangeResp := new(pb.Exchange) err = proto.Unmarshal(resp1, exchangeResp) if err != nil { return err } var theirHandshake bytes.Buffer theirHandshake.Write(resp) theirHandshake.Write(encoded) theirHandshake.Write(exchangeResp.GetEpubkey()) // u.POut("Remote Peer Identified as %s\n", s.remote) ok, err := s.remote.PubKey().Verify(theirHandshake.Bytes(), exchangeResp.GetSignature()) if err != nil { return err } if !ok { return errors.New("Bad signature!") } secret, err := genSharedKey(exchangeResp.GetEpubkey()) if err != nil { return err } cmp := bytes.Compare(myPubKey, proposeResp.GetPubkey()) mIV, tIV, mCKey, tCKey, mMKey, tMKey := ci.KeyStretcher(cmp, cipherType, hashType, secret) go s.handleSecureIn(hashType, cipherType, tIV, tCKey, tMKey) go s.handleSecureOut(hashType, cipherType, mIV, mCKey, mMKey) finished := []byte("Finished") // send finished msg select { case <-s.ctx.Done(): return ErrClosed case s.Out <- finished: } // recv finished msg var resp2 []byte select { case <-s.ctx.Done(): return ErrClosed case resp2 = <-s.In: } if bytes.Compare(resp2, finished) != 0 { return fmt.Errorf("Negotiation failed, got: %s", resp2) } log.Debugf("%s handshake: Got node id: %s", s.local, s.remote) return nil }
// If less than K nodes are in the entire network, it should fail when we make // a GET rpc and nobody has the value func TestLessThanKResponses(t *testing.T) { // t.Skip("skipping test because it makes a lot of output") ctx := context.Background() u.Debug = false fn := &fauxNet{} fs := &fauxSender{} local := makePeer(nil) peerstore := peer.NewPeerstore() peerstore.Add(local) d := NewDHT(ctx, local, peerstore, fn, fs, ds.NewMapDatastore()) var ps []peer.Peer for i := 0; i < 5; i++ { ps = append(ps, _randPeer()) d.Update(ctx, ps[i]) } other := _randPeer() // Reply with random peers to every message fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage { pmes := new(pb.Message) err := proto.Unmarshal(mes.Data(), pmes) if err != nil { t.Fatal(err) } switch pmes.GetType() { case pb.Message_GET_VALUE: resp := &pb.Message{ Type: pmes.Type, CloserPeers: pb.PeersToPBPeers([]peer.Peer{other}), } mes, err := msg.FromObject(mes.Peer(), resp) if err != nil { t.Error(err) } return mes default: panic("Shouldnt recieve this.") } }) ctx, _ = context.WithTimeout(ctx, time.Second*30) _, err := d.GetValue(ctx, u.Key("hello")) if err != nil { switch err { case routing.ErrNotFound: //Success! return case u.ErrTimeout: t.Fatal("Should not have gotten timeout!") default: t.Fatalf("Got unexpected error: %s", err) } } t.Fatal("Expected to recieve an error.") }
func TestNotFound(t *testing.T) { if testing.Short() { t.SkipNow() } ctx := context.Background() fn := &fauxNet{} fs := &fauxSender{} local := makePeer(nil) peerstore := peer.NewPeerstore() peerstore.Add(local) d := NewDHT(ctx, local, peerstore, fn, fs, ds.NewMapDatastore()) var ps []peer.Peer for i := 0; i < 5; i++ { ps = append(ps, _randPeer()) d.Update(ctx, ps[i]) } // Reply with random peers to every message fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage { pmes := new(pb.Message) err := proto.Unmarshal(mes.Data(), pmes) if err != nil { t.Fatal(err) } switch pmes.GetType() { case pb.Message_GET_VALUE: resp := &pb.Message{Type: pmes.Type} peers := []peer.Peer{} for i := 0; i < 7; i++ { peers = append(peers, _randPeer()) } resp.CloserPeers = pb.PeersToPBPeers(peers) mes, err := msg.FromObject(mes.Peer(), resp) if err != nil { t.Error(err) } return mes default: panic("Shouldnt recieve this.") } }) ctx, _ = context.WithTimeout(ctx, time.Second*5) v, err := d.GetValue(ctx, u.Key("hello")) log.Debugf("get value got %v", v) if err != nil { switch err { case routing.ErrNotFound: //Success! return case u.ErrTimeout: t.Fatal("Should not have gotten timeout!") default: t.Fatalf("Got unexpected error: %s", err) } } t.Fatal("Expected to recieve an error.") }
func TestGetFailures(t *testing.T) { if testing.Short() { t.SkipNow() } ctx := context.Background() fn := &fauxNet{} fs := &fauxSender{} peerstore := peer.NewPeerstore() local := makePeer(nil) d := NewDHT(ctx, local, peerstore, fn, fs, ds.NewMapDatastore()) other := makePeer(nil) d.Update(ctx, other) // This one should time out // u.POut("Timout Test\n") ctx1, _ := context.WithTimeout(context.Background(), time.Second) _, err := d.GetValue(ctx1, u.Key("test")) if err != nil { if err != context.DeadlineExceeded { t.Fatal("Got different error than we expected", err) } } else { t.Fatal("Did not get expected error!") } // u.POut("NotFound Test\n") // Reply with failures to every message fs.AddHandler(func(mes msg.NetMessage) msg.NetMessage { pmes := new(pb.Message) err := proto.Unmarshal(mes.Data(), pmes) if err != nil { t.Fatal(err) } resp := &pb.Message{ Type: pmes.Type, } m, err := msg.FromObject(mes.Peer(), resp) return m }) // This one should fail with NotFound ctx2, _ := context.WithTimeout(context.Background(), time.Second) _, err = d.GetValue(ctx2, u.Key("test")) if err != nil { if err != routing.ErrNotFound { t.Fatalf("Expected ErrNotFound, got: %s", err) } } else { t.Fatal("expected error, got none.") } fs.handlers = nil // Now we test this DHT's handleGetValue failure typ := pb.Message_GET_VALUE str := "hello" rec, err := d.makePutRecord(u.Key(str), []byte("blah")) if err != nil { t.Fatal(err) } req := pb.Message{ Type: &typ, Key: &str, Record: rec, } // u.POut("handleGetValue Test\n") mes, err := msg.FromObject(other, &req) if err != nil { t.Error(err) } mes = d.HandleMessage(ctx, mes) pmes := new(pb.Message) err = proto.Unmarshal(mes.Data(), pmes) if err != nil { t.Fatal(err) } if pmes.GetRecord() != nil { t.Fatal("shouldnt have value") } if pmes.GetProviderPeers() != nil { t.Fatal("shouldnt have provider peers") } }
func (dht *IpfsDHT) handleGetValue(p peer.Peer, pmes *pb.Message) (*pb.Message, error) { log.Debugf("%s handleGetValue for key: %s\n", dht.self, pmes.GetKey()) // setup response resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel()) // first, is the key even a key? key := pmes.GetKey() if key == "" { return nil, errors.New("handleGetValue but no key was provided") } // let's first check if we have the value locally. log.Debugf("%s handleGetValue looking into ds", dht.self) dskey := u.Key(pmes.GetKey()).DsKey() iVal, err := dht.datastore.Get(dskey) log.Debugf("%s handleGetValue looking into ds GOT %v", dht.self, iVal) // if we got an unexpected error, bail. if err != nil && err != ds.ErrNotFound { return nil, err } // Note: changed the behavior here to return _as much_ info as possible // (potentially all of {value, closer peers, provider}) // if we have the value, send it back if err == nil { log.Debugf("%s handleGetValue success!", dht.self) byts, ok := iVal.([]byte) if !ok { return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey) } rec := new(pb.Record) err := proto.Unmarshal(byts, rec) if err != nil { log.Error("Failed to unmarshal dht record from datastore") return nil, err } resp.Record = rec } // if we know any providers for the requested value, return those. provs := dht.providers.GetProviders(u.Key(pmes.GetKey())) if len(provs) > 0 { log.Debugf("handleGetValue returning %d provider[s]", len(provs)) resp.ProviderPeers = pb.PeersToPBPeers(provs) } // Find closest peer on given cluster to desired key and reply with that info closer := dht.betterPeersToQuery(pmes, CloserPeerCount) if closer != nil { for _, p := range closer { log.Debugf("handleGetValue returning closer peer: '%s'", p) if len(p.Addresses()) < 1 { log.Critical("no addresses on peer being sent!") } } resp.CloserPeers = pb.PeersToPBPeers(closer) } return resp, nil }
func (s *Node) loadData() error { s.cached = new(ftpb.Data) return proto.Unmarshal(s.Nd.Data, s.cached) }